diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 4bf696e..c8405bc 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -45,23 +45,18 @@ jobs: run: | git config --global user.email "dummy@email.com" git config --global user.name "Dummy User" - - name: install dependencies run: | sudo apt-get update sudo apt-get install clang qt6-base-dev libglvnd-dev libeigen3-dev zlib1g-dev libfftw3-dev ninja-build - - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.3 - - name: Get CMake uses: lukka/get-cmake@latest with: cmakeVersion: '3.16.3' - - name: Print CMake version run: cmake --version - - name: Clone latest MRtrix and switch to latest tag run: | mkdir $MRTRIX_HOME @@ -71,10 +66,8 @@ jobs: git checkout pydra-usage-python-cmd git tag -a $MRTRIX_VERSION -m"Tag used to create a pydra-mrtrix3 release" git describe --abbrev=0 - # echo "MRTRIX_VERSION=$(git describe --tags --abbrev=0)" >> $GITHUB_ENV # git checkout $MRTRIX_VERSION - - name: configure run: | cd $MRTRIX_HOME/src @@ -88,40 +81,31 @@ jobs: -D CMAKE_C_COMPILER=clang \ -D CMAKE_CXX_COMPILER=clang++ \ -D CMAKE_INSTALL_PREFIX=$MRTRIX_INSTALL - - name: Build Mrtrix run: | cd $MRTRIX_HOME/src cmake --build build - - name: Install Mrtrix run: | cd $MRTRIX_HOME/src cmake --install build - - name: Set PATH Variable run: echo "PATH=$PATH:$MRTRIX_INSTALL/bin" >> $GITHUB_ENV - - name: Set LD_LIBRARY_PATH Variable run: echo "LD_LIBRARY_PATH=$MRTRIX_INSTALL/lib" >> $GITHUB_ENV - - name: Change back to the root directory run: cd .. - - name: Set up Python uses: actions/setup-python@v2 - - name: Install Python build dependencies run: | python -m pip install --upgrade pip - - name: Install pydra-auto-gen requirements run: > pip install -e related-packages/fileformats -e related-packages/fileformats-extras -e .[dev,test] - - name: Generate task specifications run: > ./generate.py @@ -130,22 +114,18 @@ jobs: $MRTRIX_VERSION --log-errors --latest - - name: Upload MRtrix3 install uses: actions/upload-artifact@v2 with: name: MRtrix3 path: ${{ env.MRTRIX_INSTALL}} - - name: Upload auto-gen pydra uses: actions/upload-artifact@v2 with: name: AutoGen path: pydra/tasks/mrtrix3/${{ env.SUBPKG_NAME }} - - name: Write version file run: echo $MRTRIX_VERSION > mrtrix3_version.txt - - name: Upload version file uses: actions/upload-artifact@v2 with: @@ -165,40 +145,32 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Download version file uses: actions/download-artifact@v2 with: name: VersionFile path: mrtrix3_version.txt - - name: Extract Mrtrix version run: echo "MRTRIX_VERSION=$(cat mrtrix3_version.txt)" >> $GITHUB_ENV - - name: Download auto-gen pydra uses: actions/download-artifact@v2 with: name: AutoGen path: pydra/tasks/mrtrix3/${{ env.SUBPKG_NAME }} - - name: Strip auto package from gitignore so it is included in package run: | sed -i '/\/pydra\/tasks\/mrtrix3\/${{ env.SUBPKG_NAME }}/d' .gitignore - - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Install build dependencies run: | python -m pip install --upgrade pip - - name: Install Pydra run: | pip install ${{ matrix.pydra }} python -c "import pydra as m; print(f'{m.__name__} {m.__version__} @ {m.__file__}')" - - name: Install task package run: | pip install ${{ matrix.pip-flags }} "related-packages/fileformats" @@ -221,66 +193,55 @@ jobs: run: | sudo apt-get update sudo apt-get install clang qt6-base-dev libglvnd-dev libeigen3-dev zlib1g-dev libfftw3-dev ninja-build - - name: Checkout code uses: actions/checkout@v2 - + - name: Unset header + # checkout@v2 adds a header that makes branch protection report errors + # because the Github action bot is not a collaborator on the repo + run: git config --local --unset http.https://github.com/.extraheader + - name: Fetch tags + run: git fetch --prune --unshallow - name: Download version file uses: actions/download-artifact@v2 with: name: VersionFile path: mrtrix3_version.txt - - name: Extract Mrtrix version run: echo "MRTRIX_VERSION=$(cat mrtrix3_version.txt)" >> $GITHUB_ENV - - name: Download auto-gen pydra uses: actions/download-artifact@v2 with: name: AutoGen path: pydra/tasks/mrtrix3/${{ env.SUBPKG_NAME }} - - name: Strip auto package from gitignore so it is included in package run: | sed -i '/\/pydra\/tasks\/mrtrix3\/${{ env.SUBPKG_NAME }}/d' .gitignore - - name: Download MRtrix3 install uses: actions/download-artifact@v2 with: name: MRtrix3 path: ${{ env.MRTRIX_INSTALL}} - - name: Make commands executable run: chmod +x ${{ env.MRTRIX_INSTALL }}/bin/* - - name: Set PATH Variable run: echo "PATH=$PATH:$MRTRIX_INSTALL/bin" >> $GITHUB_ENV - - name: Set LD_LIBRARY_PATH Variable run: echo "LD_LIBRARY_PATH=$MRTRIX_INSTALL/lib" >> $GITHUB_ENV - - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Install build dependencies run: | python -m pip install --upgrade pip - - name: Install task package run: > pip install -e ./related-packages/fileformats -e ./related-packages/fileformats-extras -e '.[test]' - - - name: Install dev branch of pydra until bugfix - run: pip install --upgrade git+https://github.com/nipype/pydra.git@typing-bugfixes - - name: Test with pytest run: pytest -sv pydra/tasks/mrtrix3 --cov pydra.tasks.mrtrix3 --cov-report xml - - uses: codecov/codecov-action@v1 if: ${{ always() }} @@ -304,7 +265,7 @@ jobs: run: twine check ./related-packages/fileformats/dist/* - name: Check for PyPI token on tag id: deployable - if: github.event_name == 'release' + if: github.event_name == 'release' || github.event_name == 'repository_dispatch' env: PYPI_API_TOKEN: "${{ secrets.PYPI_FILEFORMATS_API_TOKEN }}" run: if [ -n "$PYPI_API_TOKEN" ]; then echo "DEPLOY=true" >> $GITHUB_OUTPUT; fi @@ -336,7 +297,7 @@ jobs: run: twine check ./related-packages/fileformats-extras/dist/* - name: Check for PyPI token on tag id: deployable - if: github.event_name == 'release' + if: github.event_name == 'release' || github.event_name == 'repository_dispatch' env: PYPI_API_TOKEN: "${{ secrets.PYPI_FILEFORMATS_EXTRAS_API_TOKEN }}" run: if [ -n "$PYPI_API_TOKEN" ]; then echo "DEPLOY=true" >> $GITHUB_OUTPUT; fi @@ -353,73 +314,58 @@ jobs: needs: [devcheck, test, deploy-fileformats, deploy-fileformats-extras] runs-on: ubuntu-latest steps: - - name: Checkout repo uses: actions/checkout@v2 - - name: Set up Git user run: | git config --local user.email "action@github.com" git config --local user.name "GitHub Action" - - name: Download version file uses: actions/download-artifact@v2 with: name: VersionFile path: mrtrix3_version.txt - - name: Extract Mrtrix version run: echo "MRTRIX_VERSION=$(cat mrtrix3_version.txt)" >> $GITHUB_ENV - - name: Download auto-gen pydra uses: actions/download-artifact@v2 with: name: AutoGen path: pydra/tasks/mrtrix3/${{ env.SUBPKG_NAME }} - - name: Strip auto package from gitignore so it is included in package run: | sed -i '/\/pydra\/tasks\/mrtrix3\/${{ env.SUBPKG_NAME }}/d' .gitignore - - name: Add auto-generated directory to git repo if: github.event_name == 'release' || github.event_name == 'repository_dispatch' run: | git add pydra/tasks/mrtrix3/$SUBPKG_NAME git commit -am"added auto-generated version to make new tag for package version" git status - - name: Get latest version tag id: latest_tag run: | git fetch --tags echo "TAG=$(git tag -l | grep 'v.*' | tail -n 1 | awk -F post '{print $1}')" >> $GITHUB_OUTPUT - - name: Overwrite the tag of release event with latest commit (i.e. including the auto directory) if: github.event_name == 'release' run: | git tag -d ${{ steps.latest_tag.outputs.TAG }}; git tag -a ${{ steps.latest_tag.outputs.TAG }} -m"Tag used to create a pydra-mrtrix3 $MRTRIX_VERSION release"; - - name: Set up Python 3.11 uses: actions/setup-python@v2 with: python-version: 3.11 - - name: Install build tools run: python -m pip install --upgrade pip twine build - - name: Build source and wheel distributions run: python -m build - - name: Check distributions run: twine check dist/* - - name: Upload sdist uses: actions/upload-artifact@v2 with: name: SDist path: dist/*.tar.gz - # Deploy on tags if PYPI_API_TOKEN is defined in the repository secrets. # Secrets are not accessible in the if: condition [0], so set an output variable [1] # [0] https://github.community/t/16928 @@ -430,7 +376,6 @@ jobs: env: PYPI_API_TOKEN: "${{ secrets.PYPI_API_TOKEN }}" run: if [ -n "$PYPI_API_TOKEN" ]; then echo ::set-output name=DEPLOY::true; fi - - name: Upload to PyPI if: steps.deployable.outputs.DEPLOY uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.gitignore b/.gitignore index c55f867..a0713aa 100644 --- a/.gitignore +++ b/.gitignore @@ -137,4 +137,7 @@ dmypy.json # Hatchling _version.py +# Mac Garbage +.DS_Store + /pydra/tasks/mrtrix3/v3_0 diff --git a/pyproject.toml b/pyproject.toml index ad2fb37..23d4949 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ description = "pydra-mrtrix3 contains Pydra task specifications for MRtrix3 tool readme = "README.rst" requires-python = ">=3.8" dependencies = [ - "fileformats-medimage_mrtrix3 >=3.0.4a4", + "fileformats-medimage_mrtrix3 >=3.0.4a5", "numpy", "pydra >=0.23", ] @@ -36,8 +36,7 @@ dev = [ "click >=8.1.3", "tqdm", "attrs >=23.1.0", - "fileformats >= 0.8", - "fileformats-extras >= 0.2.1", + "fileformats-extras >= 0.12.1", ] doc = [ "packaging", diff --git a/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/converters.py b/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/converters.py index 1301188..c87918c 100644 --- a/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/converters.py +++ b/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/converters.py @@ -1,4 +1,4 @@ -from fileformats.core import hook +from fileformats.core import converter from fileformats.medimage.base import MedicalImage from fileformats.medimage_mrtrix3 import ( @@ -16,7 +16,7 @@ in_out_file_kwargs = {} -@hook.converter( +@converter( source_format=MedicalImage, target_format=MrtrixImage, out_ext=MrtrixImage.ext, @@ -40,7 +40,7 @@ def mrconvert(name, out_ext: str, **kwargs): return MrConvert(name=name, out_file="out" + out_ext, **kwargs) -@hook.converter( +@converter( source_format=MedicalImage, target_format=MrtrixImageHeader, out_ext=MrtrixImageHeader.ext, diff --git a/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/gradients.py b/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/gradients.py index 4b9f9b5..e25d5da 100644 --- a/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/gradients.py +++ b/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/gradients.py @@ -1,9 +1,10 @@ import numpy as np +from fileformats.core import extra_implementation from fileformats.medimage import DwiEncoding from fileformats.medimage_mrtrix3 import BFile -@DwiEncoding.read_array.register +@extra_implementation(DwiEncoding.read_array) def bfile_read_array(bfile: BFile) -> np.ndarray: return np.asarray( [[float(x) for x in ln.split()] for ln in bfile.read_contents().splitlines()] diff --git a/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/image.py b/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/image.py index 7f58e5b..be543ee 100644 --- a/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/image.py +++ b/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/image.py @@ -4,23 +4,23 @@ from pathlib import Path import numpy as np from medimages4tests.dummy.nifti import get_image as get_dummy_nifti -from fileformats.core import FileSet, SampleFileGenerator +from fileformats.core import FileSet, SampleFileGenerator, extra_implementation from fileformats.medimage import MedicalImage, Nifti1 from fileformats.medimage_mrtrix3 import ImageFormat -@FileSet.generate_sample_data.register +@extra_implementation(FileSet.generate_sample_data) def generate_mrtrix_sample_data( mif: ImageFormat, generator: SampleFileGenerator, -) -> ty.Iterable[Path]: +) -> ty.List[Path]: nifti = Nifti1(get_dummy_nifti(generator.dest_dir / "nifti.nii")) with mock.patch.dict(os.environ, {"MRTRIX_CLI_PARSE_ONLY": "0"}): mif = ImageFormat.convert(nifti) return mif.fspaths -@MedicalImage.read_array.register +@extra_implementation(MedicalImage.read_array) def mrtrix_read_array(mif: ImageFormat) -> np.ndarray: raise NotImplementedError( "Need to work out how to use the metadata to read the array in the correct order" diff --git a/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/tracks.py b/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/tracks.py index a7594b6..2ef9f37 100644 --- a/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/tracks.py +++ b/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/tracks.py @@ -3,15 +3,15 @@ import math from pathlib import Path import typing as ty -from fileformats.core import FileSet, SampleFileGenerator +from fileformats.core import FileSet, SampleFileGenerator, extra_implementation from fileformats.medimage_mrtrix3 import Tracks -@FileSet.generate_sample_data.register +@extra_implementation(FileSet.generate_sample_data) def generate_tracks_sample_data( tracks: Tracks, generator: SampleFileGenerator, -) -> ty.Iterable[Path]: +) -> ty.List[Path]: """Generate a tracks file with a single straight track of length 10""" fspath = generator.dest_dir / "tracks.tck" timestamp = str(time.time() * 1e9 + time.process_time_ns()) diff --git a/related-packages/fileformats-extras/pyproject.toml b/related-packages/fileformats-extras/pyproject.toml index 7155dad..0ee7929 100644 --- a/related-packages/fileformats-extras/pyproject.toml +++ b/related-packages/fileformats-extras/pyproject.toml @@ -9,7 +9,7 @@ readme = "README.rst" requires-python = ">=3.8" dependencies = [ "fileformats >= 0.8", - "fileformats-medimage-mrtrix3 >=3.0.4a4", + "fileformats-medimage-mrtrix3 >=3.0.4a5", "medimages4tests", "pydra >= 0.23.0", ] diff --git a/related-packages/fileformats/fileformats/medimage_mrtrix3/dwi.py b/related-packages/fileformats/fileformats/medimage_mrtrix3/dwi.py index 74bbcff..287e9da 100644 --- a/related-packages/fileformats/fileformats/medimage_mrtrix3/dwi.py +++ b/related-packages/fileformats/fileformats/medimage_mrtrix3/dwi.py @@ -1,4 +1,3 @@ -from fileformats.core import hook from fileformats.core.mixin import WithAdjacentFiles from fileformats.medimage import DwiEncoding, Nifti1, NiftiGz, NiftiX, NiftiGzX from .image import ImageFormat, ImageHeader, ImageFormatGz @@ -12,7 +11,6 @@ class BFile(DwiEncoding): # NIfTI file format gzipped with BIDS side car class WithBFile(WithAdjacentFiles): - @hook.required @property def encoding(self) -> BFile: return BFile(self.select_by_ext(BFile)) @@ -35,12 +33,12 @@ class NiftiGzXB(WithBFile, NiftiGzX): class ImageFormatB(WithBFile, ImageFormat): - pass + iana_mime = "application/x-mrtrix-image-format.b" class ImageFormatGzB(WithBFile, ImageFormatGz): - pass + iana_mime = "application/x-mrtrix-image-format+gzip.b" class ImageHeaderB(WithBFile, ImageHeader): - pass + iana_mime = "application/x-mrtrix-image-header.b" diff --git a/related-packages/fileformats/fileformats/medimage_mrtrix3/image.py b/related-packages/fileformats/fileformats/medimage_mrtrix3/image.py index 1a5c74c..1b4b587 100644 --- a/related-packages/fileformats/fileformats/medimage_mrtrix3/image.py +++ b/related-packages/fileformats/fileformats/medimage_mrtrix3/image.py @@ -1,6 +1,6 @@ -import typing as ty from pathlib import Path -from fileformats.core import hook +import typing as ty +from fileformats.core import FileSet, extra_implementation from fileformats.generic import File from fileformats.application import Gzip from fileformats.core.mixin import WithMagicNumber @@ -17,43 +17,6 @@ class BaseMrtrixImage(WithMagicNumber, fileformats.medimage.MedicalImage, File): magic_number = b"mrtrix image\n" binary = True - def read_metadata(self): - metadata = {} - with open(self.fspath, "rb") as f: - line = f.readline() - if line != self.magic_number: - raise FormatMismatchError( - f"Magic line {line} doesn't match reference {self.magic_number}" - ) - line = f.readline().decode("utf-8") - while line and line != "END\n": - key, value = line.split(": ", maxsplit=1) - if "," in value: - try: - value = [int(v) for v in value.split(",")] - except ValueError: - try: - value = [float(v) for v in value.split(",")] - except ValueError: - pass - else: - try: - value = int(value) - except ValueError: - try: - value = float(value) - except ValueError: - pass - if key in metadata: - if isinstance(metadata[key], MultiLineMetadataValue): - metadata[key].append(value) - else: - metadata[key] = MultiLineMetadataValue([metadata[key], value]) - else: - metadata[key] = value - line = f.readline().decode("utf-8") - return metadata - @property def data_fspath(self): data_fspath = self.metadata["file"].split()[0] @@ -67,7 +30,9 @@ def data_fspath(self): @property def data_offset(self): - return int(self.metadata["file"].split()[1]) + fspath_and_offset = self.metadata["file"].split() + assert len(fspath_and_offset) <= 2 + return int(fspath_and_offset[1]) if len(fspath_and_offset) > 1 else 0 @property def vox_sizes(self): @@ -81,8 +46,9 @@ def dims(self): class ImageFormat(BaseMrtrixImage): ext = ".mif" + iana_mime = "application/x-mrtrix-image-format" - @hook.check + @property def check_data_file(self): if self.data_fspath != self.fspath: raise FormatMismatchError( @@ -97,15 +63,15 @@ def data_file(self): class ImageFormatGz(Gzip[ImageFormat]): - iana_mime = "application/x-image-format-gz" + iana_mime = "application/x-mrtrix-image-format-gz" ext = ".mif.gz" class ImageHeader(BaseMrtrixImage): ext = ".mih" + iana_mime = "application/x-mrtrix-image-header" - @hook.required @property def data_file(self): return ImageDataFile(self.data_fspath) @@ -120,3 +86,46 @@ def __attrs_post_init__(self): class ImageDataFile(File): ext = ".dat" + + +@extra_implementation(FileSet.read_metadata) +def mrtrix_read_metadata( + mif: BaseMrtrixImage, selected_keys: ty.Optional[ty.Collection[str]] = None +) -> ty.Mapping[str, ty.Any]: + metadata = {} + with open(mif.fspath, "rb") as f: + line = f.readline() + if line != mif.magic_number: + raise FormatMismatchError( + f"Magic line {line} doesn't match reference {mif.magic_number}" + ) + line = f.readline().decode("utf-8") + while line and line != "END\n": + key, value = line.split(": ", maxsplit=1) + if "," in value: + try: + value = [int(v) for v in value.split(",")] + except ValueError: + try: + value = [float(v) for v in value.split(",")] + except ValueError: + pass + else: + try: + value = int(value) + except ValueError: + try: + value = float(value) + except ValueError: + pass + if key in metadata: + if isinstance(metadata[key], MultiLineMetadataValue): + metadata[key].append(value) + else: + metadata[key] = MultiLineMetadataValue([metadata[key], value]) + else: + metadata[key] = value + line = f.readline().decode("utf-8") + if selected_keys: + metadata = {k: v for k, v in metadata.items() if k in selected_keys} + return metadata