refactor(tests): update dataset preparation fixtures and improve path…

… handling

.gitignore

@@ -229,3 +229,4 @@ data/**
 
 test_data
 
+*.lock

tests/conftest.py

@@ -4,47 +4,67 @@
 from zipfile import ZipFile
 
 import pytest
+from filelock import FileLock
 
 from imgtools.logging import logger
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="package")
 def curr_path():
-    return pathlib.Path(__file__).parent.parent.resolve().as_posix()
+    return pathlib.Path().cwd().resolve().absolute()
+
+
+@pytest.fixture(scope="session")
+def prepare_dataset():
+    """Prepares the dataset if not already downloaded."""
+    curr_path = pathlib.Path().cwd().resolve().absolute()
+    quebec_path = pathlib.Path(curr_path, "data", "Head-Neck-PET-CT").absolute()
+
+    # when running xdist, use lockfile to prevent all processors from trying to download the dataset
+    lock_path = pathlib.Path(curr_path, ".dataset.lock")
 
+    with FileLock(lock_path):
+        logger.info(
+            "Checking if the test dataset is downloaded...",
+            curr_path=curr_path,
+            quebec_path=quebec_path,
+        )
+        if not (quebec_path.exists() and len(list(quebec_path.glob("*"))) == 2):
+            quebec_path.mkdir(parents=True, exist_ok=True)
 
-@pytest.fixture(scope='package')
-def dataset_path(curr_path):
-    quebec_path = pathlib.Path(curr_path, "data", "Head-Neck-PET-CT")
+            # Download QC dataset
+            logger.info("Downloading the test dataset...")
+            quebec_data_url = "https://github.com/bhklab/tcia_samples/blob/main/Head-Neck-PET-CT.zip?raw=true"
+            quebec_zip_path = pathlib.Path(
+                quebec_path, "Head-Neck-PET-CT.zip"
+            ).as_posix()
+            request.urlretrieve(quebec_data_url, quebec_zip_path)
+            with ZipFile(quebec_zip_path, "r") as zipfile:
+                zipfile.extractall(quebec_path)
+            os.remove(quebec_zip_path)
+        else:
+            logger.info("Data already downloaded...")
 
-    if not (quebec_path.exists() and len(list(quebec_path.glob("*"))) == 2):
-        quebec_path.mkdir(parents=True, exist_ok=True)
+    yield quebec_path
 
-        # Download QC dataset
-        logger.info("Downloading the test dataset...")
-        quebec_data_url = "https://github.com/bhklab/tcia_samples/blob/main/Head-Neck-PET-CT.zip?raw=true"
-        quebec_zip_path = pathlib.Path(quebec_path, "Head-Neck-PET-CT.zip").as_posix()
-        request.urlretrieve(quebec_data_url, quebec_zip_path)
-        with ZipFile(quebec_zip_path, "r") as zipfile:
-            zipfile.extractall(quebec_path)
-        # os.remove(quebec_zip_path)
-    else:
-        logger.info("Data already downloaded...")
+    # Delete the lock file
+    if lock_path.exists():
+        lock_path.unlink()
 
+
+@pytest.fixture(scope="package")
+def dataset_path(prepare_dataset):
+    """Provides paths related to the dataset for tests."""
+    curr_path = pathlib.Path().cwd().resolve().absolute()
     output_path = pathlib.Path(curr_path, "tests", "temp").as_posix()
-    quebec_path = quebec_path.as_posix()
 
-    # Dataset name
+    # Paths
+    quebec_path = prepare_dataset.as_posix()
     dataset_name = os.path.basename(quebec_path)
     imgtools_path = pathlib.Path(os.path.dirname(quebec_path), ".imgtools")
-
-    # Defining paths for autopipeline and dataset component
     crawl_path = pathlib.Path(imgtools_path, f"imgtools_{dataset_name}.csv").as_posix()
     edge_path = pathlib.Path(
         imgtools_path, f"imgtools_{dataset_name}_edges.csv"
     ).as_posix()
-    # json_path =  pathlib.Path(imgtools_path, f"imgtools_{dataset_name}.json").as_posix()  # noqa: F841
 
     yield quebec_path, output_path, crawl_path, edge_path
-
-

tests/test_modalities.py

@@ -12,10 +12,9 @@
 from imgtools.io import read_dicom_auto
 from imgtools.ops import StructureSetToSegmentation
 
-
-@pytest.fixture(scope="module")
-def modalities_path(curr_path, dataset_path):
-    qc_path = pathlib.Path(curr_path, "data", "Head-Neck-PET-CT", "HN-CHUS-052")
+@pytest.fixture
+def modalities_path(dataset_path):
+    qc_path = pathlib.Path(dataset_path[0]) / "HN-CHUS-052"
     assert qc_path.exists(), "Dataset not found"
 
     path = {}
@@ -40,7 +39,8 @@ def modalities_path(curr_path, dataset_path):
     return path
 
 
-@pytest.mark.xdist_group("modalities")
+
+
 @pytest.mark.parametrize("imaging_modality", ["CT", "RTSTRUCT", "RTDOSE", "PT"])
 def test_modalities(
     imaging_modality, modalities_path