From 9ad5456676837c23d4ea38a0231b273f9667097b Mon Sep 17 00:00:00 2001 From: Jermiah Date: Fri, 17 Jan 2025 18:08:25 +0000 Subject: [PATCH 01/13] feat: update __init__.py to explicitly define exported members --- src/imgtools/ops/__init__.py | 42 +++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/src/imgtools/ops/__init__.py b/src/imgtools/ops/__init__.py index 6965f896..a79778a4 100644 --- a/src/imgtools/ops/__init__.py +++ b/src/imgtools/ops/__init__.py @@ -1 +1,41 @@ -from .ops import * +from .ops import ( + BaseInput, + BaseOp, + BaseOutput, + CentreCrop, + ClipIntensity, + Crop, + HDF5Output, + ImageAutoInput, + ImageAutoOutput, + ImageStatistics, + MinMaxScale, + NumpyOutput, + Resample, + Resize, + StandardScale, + StructureSetToSegmentation, + WindowIntensity, + Zoom, +) + +__all__ = [ + "ImageAutoInput", + "ImageAutoOutput", + "StructureSetToSegmentation", + "BaseOp", + "BaseInput", + "BaseOutput", + "CentreCrop", + "ClipIntensity", + "Crop", + "HDF5Output", + "ImageStatistics", + "MinMaxScale", + "NumpyOutput", + "Resample", + "Resize", + "StandardScale", + "WindowIntensity", + "Zoom", +] From b8c468d0ce38564835b45d870b40d6ec0a893f1a Mon Sep 17 00:00:00 2001 From: Jermiah Date: Fri, 17 Jan 2025 18:09:58 +0000 Subject: [PATCH 02/13] refactor(tests): update test_ops.py to use tmp_path and improve output path handling --- tests/dicom/sort/test_sort_utils.py | 173 +++++++++++++++------------- tests/test_ops.py | 53 +++++---- 2 files changed, 122 insertions(+), 104 deletions(-) diff --git a/tests/dicom/sort/test_sort_utils.py b/tests/dicom/sort/test_sort_utils.py index 9203bf72..cf03e0b5 100644 --- a/tests/dicom/sort/test_sort_utils.py +++ b/tests/dicom/sort/test_sort_utils.py @@ -10,48 +10,48 @@ from imgtools.dicom.sort.utils import read_tags, sanitize_file_name, truncate_uid -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def dicom_test_file(): """Pytest fixture to create a DICOM file for testing.""" ds = Dataset() # Patient and instance-related metadata - ds.PatientName = 'Test^Firstname' - ds.PatientID = '123456' + ds.PatientName = "Test^Firstname" + ds.PatientID = "123456" # Dates and times datetime.datetime.now() - ds.ContentDate = '20021114' - ds.ContentTime = '111131' # Updated time format + ds.ContentDate = "20021114" + ds.ContentTime = "111131" # Updated time format # Other metadata - ds.SpecificCharacterSet = 'ISO_IR 100' - ds.ImageType = ['ORIGINAL', 'PRIMARY', 'AXIAL'] - ds.StudyDate = '20021114' - ds.SeriesDate = '20021114' - ds.AcquisitionDate = '20021114' - ds.StudyTime = '105444' - ds.SeriesTime = '110039' - ds.AcquisitionTime = '110234.982284' - ds.Modality = 'CT' - ds.Manufacturer = 'GE MEDICAL SYSTEMS' - ds.StudyDescription = 'CT ABD & PELVIS W/O &' - ds.SeriesDescription = '2.5SOFT + 30%ASIR' - ds.ManufacturerModelName = 'LightSpeed VCT' - ds.SeriesInstanceUID = UID('1.2.840.113619.2.55.3.604688.12345678.1234567890') - ds.StudyInstanceUID = UID('1.2.840.113619.2.55.3.604688.98765432.9876543210') + ds.SpecificCharacterSet = "ISO_IR 100" + ds.ImageType = ["ORIGINAL", "PRIMARY", "AXIAL"] + ds.StudyDate = "20021114" + ds.SeriesDate = "20021114" + ds.AcquisitionDate = "20021114" + ds.StudyTime = "105444" + ds.SeriesTime = "110039" + ds.AcquisitionTime = "110234.982284" + ds.Modality = "CT" + ds.Manufacturer = "GE MEDICAL SYSTEMS" + ds.StudyDescription = "CT ABD & PELVIS W/O &" + ds.SeriesDescription = "2.5SOFT + 30%ASIR" + ds.ManufacturerModelName = "LightSpeed VCT" + ds.SeriesInstanceUID = UID("1.2.840.113619.2.55.3.604688.12345678.1234567890") + ds.StudyInstanceUID = UID("1.2.840.113619.2.55.3.604688.98765432.9876543210") # File meta information file_meta = FileMetaDataset() - file_meta.MediaStorageSOPClassUID = UID('1.2.840.10008.5.1.4.1.1.2') + file_meta.MediaStorageSOPClassUID = UID("1.2.840.10008.5.1.4.1.1.2") file_meta.MediaStorageSOPInstanceUID = UID( - '1.3.6.1.4.1.14519.5.2.1.1706.4016.292639580813240923432069920621' + "1.3.6.1.4.1.14519.5.2.1.1706.4016.292639580813240923432069920621" ) - file_meta.ImplementationClassUID = UID('1.2.3.4') + file_meta.ImplementationClassUID = UID("1.2.3.4") file_meta.TransferSyntaxUID = ExplicitVRLittleEndian ds.file_meta = file_meta # Create a temporary file - temp_file = Path(tempfile.NamedTemporaryFile(suffix='.dcm', delete=False).name) + temp_file = Path(tempfile.NamedTemporaryFile(suffix=".dcm", delete=False).name) ds.save_as(temp_file, write_like_original=False) yield temp_file @@ -63,53 +63,57 @@ def dicom_test_file(): class TestReadTags: def test_read_tags_with_main_metadata(self, dicom_test_file) -> None: tags = [ - 'PatientName', - 'PatientID', - 'ContentDate', - 'ContentTime', - 'SpecificCharacterSet', - 'StudyDate', - 'SeriesDate', - 'AcquisitionDate', - 'StudyTime', - 'SeriesTime', - 'AcquisitionTime', - 'Modality', - 'Manufacturer', - 'StudyDescription', - 'SeriesDescription', - 'ManufacturerModelName', + "PatientName", + "PatientID", + "ContentDate", + "ContentTime", + "SpecificCharacterSet", + "StudyDate", + "SeriesDate", + "AcquisitionDate", + "StudyTime", + "SeriesTime", + "AcquisitionTime", + "Modality", + "Manufacturer", + "StudyDescription", + "SeriesDescription", + "ManufacturerModelName", ] - result = read_tags(file=dicom_test_file, tags=tags, truncate=True, sanitize=True) - assert result['PatientName'] == sanitize_file_name('Test^Firstname') - assert result['PatientID'] == sanitize_file_name('123456') - assert result['ContentDate'] == sanitize_file_name('20021114') - assert result['ContentTime'] == sanitize_file_name('111131') - assert result['SpecificCharacterSet'] == sanitize_file_name('ISO_IR 100') - assert result['StudyDate'] == sanitize_file_name('20021114') - assert result['SeriesDate'] == sanitize_file_name('20021114') - assert result['AcquisitionDate'] == sanitize_file_name('20021114') - assert result['StudyTime'] == sanitize_file_name('105444') - assert result['SeriesTime'] == sanitize_file_name('110039') - assert result['AcquisitionTime'] == sanitize_file_name('110234.982284') - assert result['Modality'] == sanitize_file_name('CT') - assert result['Manufacturer'] == sanitize_file_name('GE MEDICAL SYSTEMS') - assert result['StudyDescription'] == sanitize_file_name('CT ABD & PELVIS W/O &') - assert result['SeriesDescription'] == sanitize_file_name('2.5SOFT + 30%ASIR') - assert result['ManufacturerModelName'] == sanitize_file_name('LightSpeed VCT') + result = read_tags( + file=dicom_test_file, tags=tags, truncate=True, sanitize=True + ) + assert result["PatientName"] == sanitize_file_name("Test^Firstname") + assert result["PatientID"] == sanitize_file_name("123456") + assert result["ContentDate"] == sanitize_file_name("20021114") + assert result["ContentTime"] == sanitize_file_name("111131") + assert result["SpecificCharacterSet"] == sanitize_file_name("ISO_IR 100") + assert result["StudyDate"] == sanitize_file_name("20021114") + assert result["SeriesDate"] == sanitize_file_name("20021114") + assert result["AcquisitionDate"] == sanitize_file_name("20021114") + assert result["StudyTime"] == sanitize_file_name("105444") + assert result["SeriesTime"] == sanitize_file_name("110039") + assert result["AcquisitionTime"] == sanitize_file_name("110234.982284") + assert result["Modality"] == sanitize_file_name("CT") + assert result["Manufacturer"] == sanitize_file_name("GE MEDICAL SYSTEMS") + assert result["StudyDescription"] == sanitize_file_name("CT ABD & PELVIS W/O &") + assert result["SeriesDescription"] == sanitize_file_name("2.5SOFT + 30%ASIR") + assert result["ManufacturerModelName"] == sanitize_file_name("LightSpeed VCT") def test_read_tags_with_empty_tags(self, dicom_test_file) -> None: tags = [] - result = read_tags(file=dicom_test_file, tags=tags, truncate=True, sanitize=True) + result = read_tags( + file=dicom_test_file, tags=tags, truncate=True, sanitize=True + ) assert result == {} def test_read_tags_with_nonexistent_tags(self, dicom_test_file) -> None: - tags = ['NonexistentTag'] + tags = ["NonexistentTag"] with pytest.raises(ValueError): read_tags(file=dicom_test_file, tags=tags, truncate=True, sanitize=True) def test_read_tags_with_partial_metadata(self, dicom_test_file) -> None: - tags = ['PatientName', 'Modality', 'NonexistentTag'] + tags = ["PatientName", "Modality", "NonexistentTag"] with pytest.raises(ValueError): read_tags(file=dicom_test_file, tags=tags, truncate=True, sanitize=True) # assert result['PatientName'] == 'Test^Firstname' @@ -117,48 +121,57 @@ def test_read_tags_with_partial_metadata(self, dicom_test_file) -> None: # assert 'NonexistentTag' not in result def test_read_tags_with_truncate_false(self, dicom_test_file) -> None: - tags = ['SeriesInstanceUID'] - result = read_tags(file=dicom_test_file, tags=tags, truncate=False, sanitize=True) - assert result['SeriesInstanceUID'] == '1.2.840.113619.2.55.3.604688.12345678.1234567890' + tags = ["SeriesInstanceUID"] + result = read_tags( + file=dicom_test_file, tags=tags, truncate=False, sanitize=True + ) + assert ( + result["SeriesInstanceUID"] + == "1.2.840.113619.2.55.3.604688.12345678.1234567890" + ) def test_read_tags_with_sanitize_false(self, dicom_test_file) -> None: - tags = ['StudyDescription'] - result = read_tags(file=dicom_test_file, tags=tags, truncate=True, sanitize=False) - assert result['StudyDescription'] == 'CT ABD & PELVIS W/O &' + tags = ["StudyDescription"] + result = read_tags( + file=dicom_test_file, tags=tags, truncate=True, sanitize=False + ) + assert result["StudyDescription"] == "CT ABD & PELVIS W/O &" def test_none_file(self) -> None: with pytest.raises(AssertionError): - read_tags(file=None, tags=['PatientID'], truncate=True, sanitize=True) + read_tags(file=None, tags=["PatientID"], truncate=True, sanitize=True) - def test_invalid_dicom_file(self) -> None: - inv_file = Path('invalid.dcm') + def test_invalid_dicom_file(self, tmp_path) -> None: + inv_file = Path(tmp_path, "invalid.dcm") inv_file.touch() with pytest.raises(InvalidDicomError): - read_tags(file=Path(inv_file), tags=['PatientID'], truncate=True, sanitize=True) + read_tags( + file=Path(inv_file), tags=["PatientID"], truncate=True, sanitize=True + ) class TestTruncateUid: def test_truncate_uid_with_default_last_digits(self) -> None: - uid = '1.2.840.10008.1.2.1' + uid = "1.2.840.10008.1.2.1" result = truncate_uid(uid) - assert result == '1.2.1' + assert result == "1.2.1" def test_truncate_uid_with_custom_last_digits(self) -> None: - uid = '1.2.840.10008.1.2.1' + uid = "1.2.840.10008.1.2.1" result = truncate_uid(uid, last_digits=10) - assert result == '0008.1.2.1' + assert result == "0008.1.2.1" def test_truncate_uid_with_short_uid(self) -> None: - uid = '12345' + uid = "12345" result = truncate_uid(uid, last_digits=10) - assert result == '12345' + assert result == "12345" def test_truncate_uid_with_zero_last_digits(self) -> None: - uid = '1.2.840.10008.1.2.1' + uid = "1.2.840.10008.1.2.1" result = truncate_uid(uid, last_digits=0) - assert result == '1.2.840.10008.1.2.1' + assert result == "1.2.840.10008.1.2.1" def test_truncate_uid_with_negative_last_digits(self) -> None: - uid = '1.2.840.10008.1.2.1' + uid = "1.2.840.10008.1.2.1" result = truncate_uid(uid, last_digits=-5) - assert result == '1.2.840.10008.1.2.1' + assert result == "1.2.840.10008.1.2.1" diff --git a/tests/test_ops.py b/tests/test_ops.py index bf3dd938..930a327a 100644 --- a/tests/test_ops.py +++ b/tests/test_ops.py @@ -1,5 +1,6 @@ import copy import pathlib +import shutil import h5py import numpy as np @@ -22,11 +23,13 @@ ) -@pytest.fixture(scope='session') -def output_path(curr_path): # curr_path is a fixture defined in conftest.py - out_path = pathlib.Path(curr_path, 'temp_outputs') - out_path.mkdir(parents=True, exist_ok=True) - return out_path +# @pytest.fixture(scope="session") +# def output_path(curr_path): # curr_path is a fixture defined in conftest.py +# out_path = pathlib.Path(curr_path, "temp_outputs") +# out_path.mkdir(parents=True, exist_ok=True) +# yield out_path +# # Cleanup after tests +# shutil.rmtree(out_path) img_shape = (100, 100, 100) @@ -42,10 +45,12 @@ def output_path(curr_path): # curr_path is a fixture defined in conftest.py class TestOutput: - @pytest.mark.parametrize('op', [NumpyOutput, HDF5Output]) # , "CT,RTDOSE,PT"]) - def test_output(self, op, output_path) -> None: + @pytest.mark.parametrize("op", [NumpyOutput, HDF5Output]) # , "CT,RTDOSE,PT"]) + def test_output(self, op, tmp_path) -> None: # get class name class_name = op.__name__ + output_path = tmp_path / "output" / class_name + output_path.mkdir(parents=True, exist_ok=True) # save output saver = op(output_path, create_dirs=False) @@ -55,13 +60,13 @@ def test_output(self, op, output_path) -> None: ).as_posix() # check output - if class_name == 'HDF5Output': - f = h5py.File(saved_path, 'r') - img = f['image'] - assert tuple(img.attrs['origin']) == origin - assert tuple(img.attrs['direction']) == direction - assert tuple(img.attrs['spacing']) == spacing - elif class_name == 'NumpyOutput': + if class_name == "HDF5Output": + f = h5py.File(saved_path, "r") + img = f["image"] + assert tuple(img.attrs["origin"]) == origin + assert tuple(img.attrs["direction"]) == direction + assert tuple(img.attrs["spacing"]) == spacing + elif class_name == "NumpyOutput": img = np.load(saved_path) # class-agnostic @@ -70,13 +75,13 @@ def test_output(self, op, output_path) -> None: class TestTransform: @pytest.mark.parametrize( - 'op,params', + "op,params", [ - (Resample, {'spacing': 3.7}), - (Resize, {'size': 10}), - (Zoom, {'scale_factor': 0.1}), - (Crop, {'crop_centre': (20, 20, 20), 'size': 10}), - (CentreCrop, {'size': 10}), + (Resample, {"spacing": 3.7}), + (Resize, {"size": 10}), + (Zoom, {"scale_factor": 0.1}), + (Crop, {"crop_centre": (20, 20, 20), "size": 10}), + (CentreCrop, {"size": 10}), ], ) def test_transform(self, op, params) -> None: @@ -100,12 +105,12 @@ def test_transform(self, op, params) -> None: class TestIntensity: @pytest.mark.parametrize( - 'op,params', + "op,params", [ - (ClipIntensity, {'lower': 0, 'upper': 500}), - (WindowIntensity, {'window': 500, 'level': 250}), + (ClipIntensity, {"lower": 0, "upper": 500}), + (WindowIntensity, {"window": 500, "level": 250}), (StandardScale, {}), - (MinMaxScale, {'minimum': 0, 'maximum': 1000}), + (MinMaxScale, {"minimum": 0, "maximum": 1000}), ], ) def test_intesity(self, op, params) -> None: From f16620e9f75effc78ed115933a75a3308663a94b Mon Sep 17 00:00:00 2001 From: Jermiah Date: Fri, 17 Jan 2025 18:10:02 +0000 Subject: [PATCH 03/13] feat(tests): enhance pytest configuration for coverage and parallel execution --- config/pytest.ini | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/config/pytest.ini b/config/pytest.ini index b72cc1a7..54383c6a 100644 --- a/config/pytest.ini +++ b/config/pytest.ini @@ -14,7 +14,7 @@ addopts = --showlocals # Generate coverage report # Tracks code coverage during test execution - --cov=imgtools + --cov # Output coverage report in terminal # Provides immediate feedback on coverage --cov-report=term-missing @@ -27,6 +27,15 @@ addopts = # Point to coverage config file # Allows customization of coverage report generation --cov-config=config/coverage.toml + # Append coverage data from previous runs + # Ensures coverage data is appended + --cov-append + # numprocessors to use for xdist plugin + # Sets number of processors to use for parallel test execution + --numprocesses=auto + # max processes + # Sets maximum number of processes to use for parallel test execution + --maxprocesses=8 # Patterns for test discovery # Defines which files are considered test files From 9ce633ce9df4c87be9861769c4b7067d9d7db799 Mon Sep 17 00:00:00 2001 From: Jermiah Date: Fri, 17 Jan 2025 18:39:14 +0000 Subject: [PATCH 04/13] chore: update coverage configuration and version to 1.14.0; add clean test task --- config/coverage.toml | 3 ++- pixi.lock | 4 ++-- pixi.toml | 4 ++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/config/coverage.toml b/config/coverage.toml index b3b991c6..235d629b 100644 --- a/config/coverage.toml +++ b/config/coverage.toml @@ -3,6 +3,7 @@ omit = [ "src/imgtools/logging/**/*.py", "src/imgtools/cli/**/*.py", "src/imgtools/dicom/index/**/*.py", + "tests/**/*.py", ] -[tool.coverage.report] \ No newline at end of file +[tool.coverage.report] diff --git a/pixi.lock b/pixi.lock index a5d36207..d259e18a 100644 --- a/pixi.lock +++ b/pixi.lock @@ -4470,8 +4470,8 @@ packages: timestamp: 1733255681319 - pypi: . name: med-imagetools - version: 1.13.0 - sha256: 42c94c083b70f7dd6f9bf59bbcafcc221e6e50f842b7f6f1701b236081c54d8b + version: 1.14.0 + sha256: 2994c6936a358b2681886c8da821df29be40f28096703fbdc76bb764ff0e9ab0 requires_dist: - h5py>=3.11.0,<4 - joblib>=1.4.2,<2 diff --git a/pixi.toml b/pixi.toml index dcfbf67a..6ea565fe 100644 --- a/pixi.toml +++ b/pixi.toml @@ -64,6 +64,10 @@ inputs = ["coverage-report/coverage.xml", "config/coverage.toml"] depends-on = ["test"] description = "Run pytest and generate coverage report" +[feature.test.tasks.clean_tests] +cmd = "rm -rf .pytest_cache ./data ./tests/temp" +description = "Clean up the test cache and data" + ############################################## DOCS ############################################### [feature.docs.dependencies] mkdocs = "*" From eaa9ac681ec0a64df97671340d61b96b8d26593f Mon Sep 17 00:00:00 2001 From: Jermiah Date: Fri, 17 Jan 2025 18:39:21 +0000 Subject: [PATCH 05/13] refactor(tests): update fixture scopes and improve path handling in conftest.py and test_modalities.py --- tests/conftest.py | 51 ++++++++++++---------------------------- tests/test_components.py | 19 ++++++++------- tests/test_modalities.py | 39 +++++++++++++++++++++++------- 3 files changed, 56 insertions(+), 53 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ec991af1..f3f2cc38 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,64 +8,43 @@ from imgtools.logging import logger -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def curr_path(): return pathlib.Path(__file__).parent.parent.resolve().as_posix() -@pytest.fixture(scope='session') +@pytest.fixture(scope='module') def dataset_path(curr_path): - quebec_path = pathlib.Path(curr_path, 'data', 'Head-Neck-PET-CT') + quebec_path = pathlib.Path(curr_path, "data", "Head-Neck-PET-CT") - if not (quebec_path.exists() and len(list(quebec_path.glob('*'))) == 2): + if not (quebec_path.exists() and len(list(quebec_path.glob("*"))) == 2): quebec_path.mkdir(parents=True, exist_ok=True) # Download QC dataset - logger.info('Downloading the test dataset...') - quebec_data_url = ( - 'https://github.com/bhklab/tcia_samples/blob/main/Head-Neck-PET-CT.zip?raw=true' - ) - quebec_zip_path = pathlib.Path(quebec_path, 'Head-Neck-PET-CT.zip').as_posix() + logger.info("Downloading the test dataset...") + quebec_data_url = "https://github.com/bhklab/tcia_samples/blob/main/Head-Neck-PET-CT.zip?raw=true" + quebec_zip_path = pathlib.Path(quebec_path, "Head-Neck-PET-CT.zip").as_posix() request.urlretrieve(quebec_data_url, quebec_zip_path) - with ZipFile(quebec_zip_path, 'r') as zipfile: + with ZipFile(quebec_zip_path, "r") as zipfile: zipfile.extractall(quebec_path) os.remove(quebec_zip_path) else: - logger.info('Data already downloaded...') + logger.info("Data already downloaded...") - output_path = pathlib.Path(curr_path, 'tests', 'temp').as_posix() + output_path = pathlib.Path(curr_path, "tests", "temp").as_posix() quebec_path = quebec_path.as_posix() # Dataset name dataset_name = os.path.basename(quebec_path) - imgtools_path = pathlib.Path(os.path.dirname(quebec_path), '.imgtools') + imgtools_path = pathlib.Path(os.path.dirname(quebec_path), ".imgtools") # Defining paths for autopipeline and dataset component - crawl_path = pathlib.Path(imgtools_path, f'imgtools_{dataset_name}.csv').as_posix() - edge_path = pathlib.Path(imgtools_path, f'imgtools_{dataset_name}_edges.csv').as_posix() + crawl_path = pathlib.Path(imgtools_path, f"imgtools_{dataset_name}.csv").as_posix() + edge_path = pathlib.Path( + imgtools_path, f"imgtools_{dataset_name}_edges.csv" + ).as_posix() # json_path = pathlib.Path(imgtools_path, f"imgtools_{dataset_name}.json").as_posix() # noqa: F841 yield quebec_path, output_path, crawl_path, edge_path -@pytest.fixture(scope='session') -def modalities_path(curr_path): - qc_path = pathlib.Path(curr_path, 'data', 'Head-Neck-PET-CT', 'HN-CHUS-052') - assert qc_path.exists(), 'Dataset not found' - - path = {} - path['CT'] = pathlib.Path( - qc_path, '08-27-1885-CA ORL FDG TEP POS TX-94629/3.000000-Merged-06362' - ).as_posix() - path['RTSTRUCT'] = pathlib.Path( - qc_path, - '08-27-1885-OrophCB.0OrophCBTRTID derived StudyInstanceUID.-94629/Pinnacle POI-41418', - ).as_posix() - path['RTDOSE'] = pathlib.Path( - qc_path, - '08-27-1885-OrophCB.0OrophCBTRTID derived StudyInstanceUID.-94629/11376', - ).as_posix() - path['PT'] = pathlib.Path( - qc_path, '08-27-1885-CA ORL FDG TEP POS TX-94629/532790.000000-LOR-RAMLA-44600' - ).as_posix() - return path diff --git a/tests/test_components.py b/tests/test_components.py index 786f9a18..94edead9 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -10,6 +10,7 @@ from imgtools.logging import logger + # @pytest.mark.parametrize("modalities",["PT", "CT,RTSTRUCT", "CT,RTDOSE", "CT,PT,RTDOSE", "CT,RTSTRUCT,RTDOSE", "CT,RTSTRUCT,RTDOSE,PT"]) @pytest.mark.xdist_group("serial") @pytest.mark.parametrize( @@ -58,23 +59,23 @@ def test_pipeline(self, modalities) -> None: pipeline.run() # Check if the crawl and edges exist - assert os.path.exists(self.crawl_path) & os.path.exists(self.edge_path), ( - "There was no crawler output" - ) + assert os.path.exists(self.crawl_path) & os.path.exists( + self.edge_path + ), "There was no crawler output" # for the test example, there are 6 files and 4 connections crawl_data = pd.read_csv(self.crawl_path, index_col=0) edge_data = pd.read_csv(self.edge_path) # this assert will fail.... - assert (len(crawl_data) == 12) & (len(edge_data) == 10), ( - "There was an error in crawling or while making the edge table" - ) + assert (len(crawl_data) == 12) & ( + len(edge_data) == 10 + ), "There was an error in crawling or while making the edge table" # Check if the dataset.csv is having the correct number of components and has all the fields comp_table = pd.read_csv(comp_path, index_col=0) - assert len(comp_table) == 2, ( - "There was some error in making components, check datagraph.parser" - ) + assert ( + len(comp_table) == 2 + ), "There was some error in making components, check datagraph.parser" # Check the nrrd files subject_id_list = list(comp_table.index) diff --git a/tests/test_modalities.py b/tests/test_modalities.py index 707cf60c..f61422df 100644 --- a/tests/test_modalities.py +++ b/tests/test_modalities.py @@ -13,38 +13,61 @@ from imgtools.ops import StructureSetToSegmentation -@pytest.mark.parametrize('modalities', ['CT', 'RTSTRUCT', 'RTDOSE', 'PT']) +@pytest.fixture(scope="module") +def modalities_path(curr_path, dataset_path): + qc_path = pathlib.Path(curr_path, "data", "Head-Neck-PET-CT", "HN-CHUS-052") + assert qc_path.exists(), "Dataset not found" + + path = {} + path["CT"] = pathlib.Path( + qc_path, "08-27-1885-CA ORL FDG TEP POS TX-94629/3.000000-Merged-06362" + ).as_posix() + path["RTSTRUCT"] = pathlib.Path( + qc_path, + "08-27-1885-OrophCB.0OrophCBTRTID derived StudyInstanceUID.-94629/Pinnacle POI-41418", + ).as_posix() + path["RTDOSE"] = pathlib.Path( + qc_path, + "08-27-1885-OrophCB.0OrophCBTRTID derived StudyInstanceUID.-94629/11376", + ).as_posix() + path["PT"] = pathlib.Path( + qc_path, "08-27-1885-CA ORL FDG TEP POS TX-94629/532790.000000-LOR-RAMLA-44600" + ).as_posix() + return path + + +@pytest.mark.parametrize("modalities", ["CT", "RTSTRUCT", "RTDOSE", "PT"]) def test_modalities( modalities, modalities_path ) -> None: # modalities_path is a fixture defined in conftest.py path = modalities_path - img = read_dicom_auto(path['CT']).image - if modalities != 'RTSTRUCT': + img = read_dicom_auto(path["CT"]).image + if modalities != "RTSTRUCT": # Checks for dimensions dcm = pydicom.dcmread( pathlib.Path(path[modalities], os.listdir(path[modalities])[0]).as_posix() ).pixel_array instances = len(os.listdir(path[modalities])) dicom = read_dicom_auto(path[modalities]) - if modalities == 'CT': + if modalities == "CT": dicom = dicom.image if instances > 1: # For comparing CT and PT modalities assert dcm.shape == (dicom.GetHeight(), dicom.GetWidth()) assert instances == dicom.GetDepth() else: # For comparing RTDOSE modalties assert dcm.shape == (dicom.GetDepth(), dicom.GetHeight(), dicom.GetWidth()) - if modalities == 'PT': + if modalities == "PT": dicom = dicom.resample_pet(img) assert dicom.GetSize() == img.GetSize() - if modalities == 'RTDOSE': + if modalities == "RTDOSE": dicom = dicom.resample_dose(img) assert dicom.GetSize() == img.GetSize() else: struc = read_dicom_auto(path[modalities]) make_binary_mask = StructureSetToSegmentation( - roi_names=['GTV.?', 'LARYNX'], continuous=False + roi_names=["GTV.?", "LARYNX"], continuous=False ) - mask = make_binary_mask(struc, img, {'background': 0}, False) + mask = make_binary_mask(struc, img, {"background": 0}, False) A = sitk.GetArrayFromImage(mask) assert len(A.shape) == 4 assert A.shape[0:3] == (img.GetDepth(), img.GetHeight(), img.GetWidth()) From 89bab4ea846143a93e71b15c1388106b92e5f57e Mon Sep 17 00:00:00 2001 From: Jermiah Date: Fri, 17 Jan 2025 18:42:38 +0000 Subject: [PATCH 06/13] refactor(tests): change fixture scope to 'package' and add xdist_group for serial execution --- tests/conftest.py | 4 ++-- tests/test_modalities.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f3f2cc38..766262e5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,7 +13,7 @@ def curr_path(): return pathlib.Path(__file__).parent.parent.resolve().as_posix() -@pytest.fixture(scope='module') +@pytest.fixture(scope='package') def dataset_path(curr_path): quebec_path = pathlib.Path(curr_path, "data", "Head-Neck-PET-CT") @@ -27,7 +27,7 @@ def dataset_path(curr_path): request.urlretrieve(quebec_data_url, quebec_zip_path) with ZipFile(quebec_zip_path, "r") as zipfile: zipfile.extractall(quebec_path) - os.remove(quebec_zip_path) + # os.remove(quebec_zip_path) else: logger.info("Data already downloaded...") diff --git a/tests/test_modalities.py b/tests/test_modalities.py index f61422df..f217d6be 100644 --- a/tests/test_modalities.py +++ b/tests/test_modalities.py @@ -35,7 +35,7 @@ def modalities_path(curr_path, dataset_path): ).as_posix() return path - +@pytest.mark.xdist_group("serial") @pytest.mark.parametrize("modalities", ["CT", "RTSTRUCT", "RTDOSE", "PT"]) def test_modalities( modalities, modalities_path From bf860dba208192924cc379ee82fc8f080b6cc8b1 Mon Sep 17 00:00:00 2001 From: Jermiah Date: Fri, 17 Jan 2025 18:51:25 +0000 Subject: [PATCH 07/13] refactor(tests): update pytest configuration for xdist group and improve test parameterization --- config/pytest.ini | 2 ++ tests/test_modalities.py | 29 ++++++++++++++++++----------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/config/pytest.ini b/config/pytest.ini index 54383c6a..39ce4555 100644 --- a/config/pytest.ini +++ b/config/pytest.ini @@ -36,6 +36,8 @@ addopts = # max processes # Sets maximum number of processes to use for parallel test execution --maxprocesses=8 + # group xdist + --dist=loadgroup # Patterns for test discovery # Defines which files are considered test files diff --git a/tests/test_modalities.py b/tests/test_modalities.py index f217d6be..f5ebe3ff 100644 --- a/tests/test_modalities.py +++ b/tests/test_modalities.py @@ -33,37 +33,44 @@ def modalities_path(curr_path, dataset_path): path["PT"] = pathlib.Path( qc_path, "08-27-1885-CA ORL FDG TEP POS TX-94629/532790.000000-LOR-RAMLA-44600" ).as_posix() + + for key, val in path.items(): + assert pathlib.Path(val).exists(), f"{key} not found at {val}" + return path -@pytest.mark.xdist_group("serial") -@pytest.mark.parametrize("modalities", ["CT", "RTSTRUCT", "RTDOSE", "PT"]) + +@pytest.mark.xdist_group("modalities") +@pytest.mark.parametrize("imaging_modality", ["CT", "RTSTRUCT", "RTDOSE", "PT"]) def test_modalities( - modalities, modalities_path + imaging_modality, modalities_path ) -> None: # modalities_path is a fixture defined in conftest.py path = modalities_path img = read_dicom_auto(path["CT"]).image - if modalities != "RTSTRUCT": + if imaging_modality != "RTSTRUCT": # Checks for dimensions dcm = pydicom.dcmread( - pathlib.Path(path[modalities], os.listdir(path[modalities])[0]).as_posix() + pathlib.Path( + path[imaging_modality], os.listdir(path[imaging_modality])[0] + ).as_posix() ).pixel_array - instances = len(os.listdir(path[modalities])) - dicom = read_dicom_auto(path[modalities]) - if modalities == "CT": + instances = len(os.listdir(path[imaging_modality])) + dicom = read_dicom_auto(path[imaging_modality]) + if imaging_modality == "CT": dicom = dicom.image if instances > 1: # For comparing CT and PT modalities assert dcm.shape == (dicom.GetHeight(), dicom.GetWidth()) assert instances == dicom.GetDepth() else: # For comparing RTDOSE modalties assert dcm.shape == (dicom.GetDepth(), dicom.GetHeight(), dicom.GetWidth()) - if modalities == "PT": + if imaging_modality == "PT": dicom = dicom.resample_pet(img) assert dicom.GetSize() == img.GetSize() - if modalities == "RTDOSE": + if imaging_modality == "RTDOSE": dicom = dicom.resample_dose(img) assert dicom.GetSize() == img.GetSize() else: - struc = read_dicom_auto(path[modalities]) + struc = read_dicom_auto(path[imaging_modality]) make_binary_mask = StructureSetToSegmentation( roi_names=["GTV.?", "LARYNX"], continuous=False ) From 02df8feb5911ff8c8dc810a6f40801c4e8d556a8 Mon Sep 17 00:00:00 2001 From: Jermiah Date: Fri, 17 Jan 2025 19:22:20 +0000 Subject: [PATCH 08/13] refactor(tests): update dataset preparation fixtures and improve path handling --- .gitignore | 1 + tests/conftest.py | 68 ++++++++++++++++++++++++++-------------- tests/test_modalities.py | 10 +++--- 3 files changed, 50 insertions(+), 29 deletions(-) diff --git a/.gitignore b/.gitignore index c59d27e3..088704f2 100644 --- a/.gitignore +++ b/.gitignore @@ -229,3 +229,4 @@ data/** test_data +*.lock \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 766262e5..cfc13940 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,47 +4,67 @@ from zipfile import ZipFile import pytest +from filelock import FileLock from imgtools.logging import logger -@pytest.fixture(scope="session") +@pytest.fixture(scope="package") def curr_path(): - return pathlib.Path(__file__).parent.parent.resolve().as_posix() + return pathlib.Path().cwd().resolve().absolute() + + +@pytest.fixture(scope="session") +def prepare_dataset(): + """Prepares the dataset if not already downloaded.""" + curr_path = pathlib.Path().cwd().resolve().absolute() + quebec_path = pathlib.Path(curr_path, "data", "Head-Neck-PET-CT").absolute() + + # when running xdist, use lockfile to prevent all processors from trying to download the dataset + lock_path = pathlib.Path(curr_path, ".dataset.lock") + with FileLock(lock_path): + logger.info( + "Checking if the test dataset is downloaded...", + curr_path=curr_path, + quebec_path=quebec_path, + ) + if not (quebec_path.exists() and len(list(quebec_path.glob("*"))) == 2): + quebec_path.mkdir(parents=True, exist_ok=True) -@pytest.fixture(scope='package') -def dataset_path(curr_path): - quebec_path = pathlib.Path(curr_path, "data", "Head-Neck-PET-CT") + # Download QC dataset + logger.info("Downloading the test dataset...") + quebec_data_url = "https://github.com/bhklab/tcia_samples/blob/main/Head-Neck-PET-CT.zip?raw=true" + quebec_zip_path = pathlib.Path( + quebec_path, "Head-Neck-PET-CT.zip" + ).as_posix() + request.urlretrieve(quebec_data_url, quebec_zip_path) + with ZipFile(quebec_zip_path, "r") as zipfile: + zipfile.extractall(quebec_path) + os.remove(quebec_zip_path) + else: + logger.info("Data already downloaded...") - if not (quebec_path.exists() and len(list(quebec_path.glob("*"))) == 2): - quebec_path.mkdir(parents=True, exist_ok=True) + yield quebec_path - # Download QC dataset - logger.info("Downloading the test dataset...") - quebec_data_url = "https://github.com/bhklab/tcia_samples/blob/main/Head-Neck-PET-CT.zip?raw=true" - quebec_zip_path = pathlib.Path(quebec_path, "Head-Neck-PET-CT.zip").as_posix() - request.urlretrieve(quebec_data_url, quebec_zip_path) - with ZipFile(quebec_zip_path, "r") as zipfile: - zipfile.extractall(quebec_path) - # os.remove(quebec_zip_path) - else: - logger.info("Data already downloaded...") + # Delete the lock file + if lock_path.exists(): + lock_path.unlink() + +@pytest.fixture(scope="package") +def dataset_path(prepare_dataset): + """Provides paths related to the dataset for tests.""" + curr_path = pathlib.Path().cwd().resolve().absolute() output_path = pathlib.Path(curr_path, "tests", "temp").as_posix() - quebec_path = quebec_path.as_posix() - # Dataset name + # Paths + quebec_path = prepare_dataset.as_posix() dataset_name = os.path.basename(quebec_path) imgtools_path = pathlib.Path(os.path.dirname(quebec_path), ".imgtools") - - # Defining paths for autopipeline and dataset component crawl_path = pathlib.Path(imgtools_path, f"imgtools_{dataset_name}.csv").as_posix() edge_path = pathlib.Path( imgtools_path, f"imgtools_{dataset_name}_edges.csv" ).as_posix() - # json_path = pathlib.Path(imgtools_path, f"imgtools_{dataset_name}.json").as_posix() # noqa: F841 yield quebec_path, output_path, crawl_path, edge_path - - diff --git a/tests/test_modalities.py b/tests/test_modalities.py index f5ebe3ff..74d2e277 100644 --- a/tests/test_modalities.py +++ b/tests/test_modalities.py @@ -12,10 +12,9 @@ from imgtools.io import read_dicom_auto from imgtools.ops import StructureSetToSegmentation - -@pytest.fixture(scope="module") -def modalities_path(curr_path, dataset_path): - qc_path = pathlib.Path(curr_path, "data", "Head-Neck-PET-CT", "HN-CHUS-052") +@pytest.fixture +def modalities_path(dataset_path): + qc_path = pathlib.Path(dataset_path[0]) / "HN-CHUS-052" assert qc_path.exists(), "Dataset not found" path = {} @@ -40,7 +39,8 @@ def modalities_path(curr_path, dataset_path): return path -@pytest.mark.xdist_group("modalities") + + @pytest.mark.parametrize("imaging_modality", ["CT", "RTSTRUCT", "RTDOSE", "PT"]) def test_modalities( imaging_modality, modalities_path From 00178ec8419a9d07d2a7988206b924a676222792 Mon Sep 17 00:00:00 2001 From: Jermiah Date: Fri, 17 Jan 2025 19:25:44 +0000 Subject: [PATCH 09/13] chore: update dependencies to include filelock version 3.16.1 and adjust dataset lock path in tests --- pixi.lock | 8 ++++++++ pixi.toml | 1 + tests/conftest.py | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pixi.lock b/pixi.lock index d259e18a..5c5f29c1 100644 --- a/pixi.lock +++ b/pixi.lock @@ -954,6 +954,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/coverage-7.6.10-py310h89163eb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.2.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/execnet-2.1.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.16.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.43-h712a8e2_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2 @@ -1034,6 +1035,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/coverage-7.6.10-py310hc74094e_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.2.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/execnet-2.1.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.16.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.2-h3422bc3_5.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/liblzma-5.6.3-h39f12f2_1.conda @@ -1114,6 +1116,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/coverage-7.6.10-py311h2dc5d0c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.2.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/execnet-2.1.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.16.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.43-h712a8e2_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.6.4-h5888daf_0.conda @@ -1195,6 +1198,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/coverage-7.6.10-py311h4921393_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.2.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/execnet-2.1.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.16.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.6.4-h286801f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.2-h3422bc3_5.tar.bz2 @@ -1276,6 +1280,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/coverage-7.6.10-py312h178313f_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.2.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/execnet-2.1.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.16.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.43-h712a8e2_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.6.4-h5888daf_0.conda @@ -1357,6 +1362,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/coverage-7.6.10-py312h998013c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.2.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/execnet-2.1.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.16.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.6.4-h286801f_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.2-h3422bc3_5.tar.bz2 @@ -1440,6 +1446,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/coverage-7.6.10-py312h178313f_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.2.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/execnet-2.1.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.16.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/h2-4.1.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hpack-4.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.0.1-pyhd8ed1ab_1.conda @@ -1548,6 +1555,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/coverage-7.6.10-py312h998013c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.2.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/execnet-2.1.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/filelock-3.16.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/h2-4.1.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hpack-4.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/hyperframe-6.0.1-pyhd8ed1ab_1.conda diff --git a/pixi.toml b/pixi.toml index 6ea565fe..feb53a4e 100644 --- a/pixi.toml +++ b/pixi.toml @@ -49,6 +49,7 @@ pytest-cov = "*" pytest-xdist = "*" pytest-mock = ">=3.14.0,<4" sqlalchemy-stubs = ">=0.4,<0.5" +filelock = ">=3.16.1,<4" [feature.test.pypi-dependencies] med-imagetools = { path = ".", editable = true } diff --git a/tests/conftest.py b/tests/conftest.py index cfc13940..e00a2076 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,7 +21,7 @@ def prepare_dataset(): quebec_path = pathlib.Path(curr_path, "data", "Head-Neck-PET-CT").absolute() # when running xdist, use lockfile to prevent all processors from trying to download the dataset - lock_path = pathlib.Path(curr_path, ".dataset.lock") + lock_path = quebec_path / ".dataset.lock" with FileLock(lock_path): logger.info( From 0d5199fe874490ad57979a44621170e73feef25a Mon Sep 17 00:00:00 2001 From: Jermiah Date: Fri, 17 Jan 2025 19:46:12 +0000 Subject: [PATCH 10/13] refactor(tests): improve test_components.py by updating path handling and modifying test parameters --- tests/test_components.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/test_components.py b/tests/test_components.py index 94edead9..38f45400 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -10,7 +10,6 @@ from imgtools.logging import logger - # @pytest.mark.parametrize("modalities",["PT", "CT,RTSTRUCT", "CT,RTDOSE", "CT,PT,RTDOSE", "CT,RTSTRUCT,RTDOSE", "CT,RTSTRUCT,RTDOSE,PT"]) @pytest.mark.xdist_group("serial") @pytest.mark.parametrize( @@ -31,19 +30,19 @@ class TestComponents: def _get_path( self, dataset_path ) -> None: # dataset_path is a fixture defined in conftest.py - self.input_path, self.output_path, self.crawl_path, self.edge_path = ( - dataset_path - ) + self.input_path, _, _, _ = dataset_path + self.input_path = pathlib.Path(self.input_path) logger.info(dataset_path) - def test_pipeline(self, modalities) -> None: + def test_pipeline(self, modalities, tmp_path) -> None: """ Testing the Autopipeline for processing the DICOMS and saving it as nrrds """ n_jobs = 2 output_path_mod = pathlib.Path( - self.output_path, str("temp_folder_" + ("_").join(modalities.split(","))) + tmp_path, str("temp_folder_" + ("_").join(modalities.split(","))) ).as_posix() + output_path_mod = pathlib.Path(output_path_mod).as_posix() # Initialize pipeline for the current setting pipeline = AutoPipeline( self.input_path, @@ -52,20 +51,27 @@ def test_pipeline(self, modalities) -> None: n_jobs=n_jobs, spacing=(5, 5, 5), overwrite=True, - update=True, + update=False, ) # Run for different modalities comp_path = pathlib.Path(output_path_mod, "dataset.csv").as_posix() pipeline.run() + dataset_name = self.input_path.name + crawl_path = ( + self.input_path.parent / ".imgtools" / f"imgtools_{dataset_name}.csv" + ) + edge_path = ( + self.input_path.parent / ".imgtools" / f"imgtools_{dataset_name}_edges.csv" + ) # Check if the crawl and edges exist - assert os.path.exists(self.crawl_path) & os.path.exists( - self.edge_path + assert os.path.exists(crawl_path) & os.path.exists( + edge_path ), "There was no crawler output" # for the test example, there are 6 files and 4 connections - crawl_data = pd.read_csv(self.crawl_path, index_col=0) - edge_data = pd.read_csv(self.edge_path) + crawl_data = pd.read_csv(crawl_path, index_col=0) + edge_data = pd.read_csv(edge_path) # this assert will fail.... assert (len(crawl_data) == 12) & ( len(edge_data) == 10 From a8522b543fdddedc01dfade1849f7ffb2fb4ea10 Mon Sep 17 00:00:00 2001 From: Jermiah Date: Fri, 17 Jan 2025 19:49:18 +0000 Subject: [PATCH 11/13] refactor(tests): remove temporary edge path after shape assertion in TestComponents --- tests/test_components.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_components.py b/tests/test_components.py index 38f45400..550ddb3a 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -108,3 +108,4 @@ def test_pipeline(self, modalities, tmp_path) -> None: shapes.append(temp_dicom.shape) A = [item == shapes[0] for item in shapes] assert all(A) + edge_path.unlink() From d7ca843fc6d5e5edd209be8f352bc8b18c0af508 Mon Sep 17 00:00:00 2001 From: Jermiah Date: Fri, 17 Jan 2025 19:57:51 +0000 Subject: [PATCH 12/13] refactor(tests): enable update flag in TestComponents for dataset processing --- tests/test_components.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_components.py b/tests/test_components.py index 550ddb3a..cf96ca10 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -51,7 +51,7 @@ def test_pipeline(self, modalities, tmp_path) -> None: n_jobs=n_jobs, spacing=(5, 5, 5), overwrite=True, - update=False, + update=True, ) # Run for different modalities comp_path = pathlib.Path(output_path_mod, "dataset.csv").as_posix() From 3d411037cc4608d80fd136015942000996a7134f Mon Sep 17 00:00:00 2001 From: Jermiah Date: Fri, 17 Jan 2025 19:59:46 +0000 Subject: [PATCH 13/13] refactor(tests): update file path in modalities_path function for consistency --- tests/test_modalities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_modalities.py b/tests/test_modalities.py index 74d2e277..69e8fe56 100644 --- a/tests/test_modalities.py +++ b/tests/test_modalities.py @@ -23,7 +23,7 @@ def modalities_path(dataset_path): ).as_posix() path["RTSTRUCT"] = pathlib.Path( qc_path, - "08-27-1885-OrophCB.0OrophCBTRTID derived StudyInstanceUID.-94629/Pinnacle POI-41418", + "08-27-1885-OrophCB.0OrophCBTRTID derived StudyInstanceUID.-94629/Pinnacle POI-41418/1-1.dcm", ).as_posix() path["RTDOSE"] = pathlib.Path( qc_path,