From 9e9227b7ef2e32ba785eeb989a187fbb008f9c93 Mon Sep 17 00:00:00 2001 From: Tom Close Date: Fri, 7 Jun 2024 12:53:30 +1000 Subject: [PATCH 1/9] touched up ci-cd --- .github/workflows/ci-cd.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 4bf696e..945c2b6 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -288,26 +288,33 @@ jobs: needs: [devcheck, test] runs-on: ubuntu-latest steps: + - uses: actions/checkout@v3 with: submodules: recursive fetch-depth: 0 + - name: Set up Python uses: actions/setup-python@v4 with: python-version: '3.11' + - name: Install build tools run: python -m pip install build twine + - name: Build source and wheel distributions run: python -m build ./related-packages/fileformats + - name: Check distributions run: twine check ./related-packages/fileformats/dist/* + - name: Check for PyPI token on tag id: deployable - if: github.event_name == 'release' + if: github.event_name == 'release' || github.event_name == 'repository_dispatch' env: PYPI_API_TOKEN: "${{ secrets.PYPI_FILEFORMATS_API_TOKEN }}" run: if [ -n "$PYPI_API_TOKEN" ]; then echo "DEPLOY=true" >> $GITHUB_OUTPUT; fi + - name: Upload to PyPI if: steps.deployable.outputs.DEPLOY uses: pypa/gh-action-pypi-publish@release/v1 @@ -320,26 +327,33 @@ jobs: needs: [deploy-fileformats] runs-on: ubuntu-latest steps: + - uses: actions/checkout@v3 with: submodules: recursive fetch-depth: 0 + - name: Set up Python uses: actions/setup-python@v4 with: python-version: '3.11' + - name: Install build tools run: python -m pip install build twine + - name: Build source and wheel distributions run: python -m build ./related-packages/fileformats-extras + - name: Check distributions run: twine check ./related-packages/fileformats-extras/dist/* + - name: Check for PyPI token on tag id: deployable - if: github.event_name == 'release' + if: github.event_name == 'release' || github.event_name == 'repository_dispatch' env: PYPI_API_TOKEN: "${{ secrets.PYPI_FILEFORMATS_EXTRAS_API_TOKEN }}" run: if [ -n "$PYPI_API_TOKEN" ]; then echo "DEPLOY=true" >> $GITHUB_OUTPUT; fi + - name: Upload to PyPI if: steps.deployable.outputs.DEPLOY uses: pypa/gh-action-pypi-publish@release/v1 From 19253420899da4bcaa1e84d9282ca09439725196 Mon Sep 17 00:00:00 2001 From: Tom Close Date: Wed, 4 Sep 2024 07:20:17 +1000 Subject: [PATCH 2/9] updated fileformats to match core refactor --- .DS_Store | Bin 0 -> 8196 bytes related-packages/.DS_Store | Bin 0 -> 6148 bytes .../extras/medimage_mrtrix3/converters.py | 6 +++--- .../extras/medimage_mrtrix3/gradients.py | 3 ++- .../fileformats/extras/medimage_mrtrix3/image.py | 6 +++--- .../extras/medimage_mrtrix3/tracks.py | 4 ++-- .../fileformats/medimage_mrtrix3/dwi.py | 8 +++----- .../fileformats/medimage_mrtrix3/image.py | 9 ++++----- 8 files changed, 17 insertions(+), 19 deletions(-) create mode 100644 .DS_Store create mode 100644 related-packages/.DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..540c295c97de9c8b4b76aa0f4a526b528aefaa40 GIT binary patch literal 8196 zcmeHMzl#$=6n>LLj0Pc{*SS?t&Ikc51o74*TKa>4V0CxNhD6Mg==}h#)<)r)1HtnO z3mZYO(f>ffJGHa7xyJSoYz6(^%$R$VWRtTIAv4F!Tjsqt@0)LSX7cukNX@TC6`~m; z%A>L@7SN0+tmisZGPdU=q=G+nT9sO(8~27ReTQejGvFEU40r}S1OE#HxU;$BQ|^7E z^{r>XGcc75@c9s+vJ9Z7sxUR!({xRSt?4*!WuZCa(e6P?a|AGVMfjvqPt_CZ!- z7#Jt+?juRJa_~^dcnZK1(|zipE>MG(P}|gWa9ztjNw<{vHsjL!XQgfCC}muUbF#IA zw-#bpItILqb%mO=j5ZB3B^SqAOWZeaoPPeTtZ|PSUt&L)hl8KKFn(UQH6U+Lm!bil zDfe^OLgM`K>T3Da-4D#~z&H|Hx|NG3z%W>+1^h9i{k7*FXZ_XAR=TCc7tNhr-!kuK z%D58eWNQbn;?0458)4UC{Eu8pu z{k|3vxk(+I3%uhZDgzIn5qHqGcn?ZcnOncU0@XwxiS^CKZt&8as|WDgKjr?$PtGgz zcoz_TnNkL_X5KTL|6fl3{y(Mc_-T6vrh@^MtJbRXn5)gLhi08YPk_3J%7yI`Bee?} pHsOAqjzjkUVTk9z>Ow0MBcVb0=N|&}?+1VXGxz@fXBMWj{08$#q;LQL literal 0 HcmV?d00001 diff --git a/related-packages/.DS_Store b/related-packages/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..0e140e22d8fea0cff70cc77dae408c61ac06cda6 GIT binary patch literal 6148 zcmeHKF;2r!47J-1K?SK|V$1>P(2*^aup{*XP|^ZYqf`YX#FPs$a0wR9!3CHYdHyzS zqqL|UKnU5A|4aPs#D0n5n25N4pY@13L=-^<8#`E5i0q5jq+u4>FaGy*Q3-|gh8H@GbFsie)_qqp^*d6A~WNil>+^pWj7KQ>>Ezw^$v z@Sa{TWs!zt_E+aiT~zRZH*bdbubTbg(>wCnC~moy*JAgpmJMSUm2Deasq^Fv>AE-r z&VV!UV+^2Yi$rZjU!4JGz!{h`Am4`=Dwr6Sis93NC5!;T7R*6#EVG2f#DOAJdz4JWqYgYC|26AFjB<$4y*7rPLq)`|RGf$4AhcrmN-I8q27x`617KoUD#8Ns OAAvxFFV4WPGVlqoS!i(p literal 0 HcmV?d00001 diff --git a/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/converters.py b/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/converters.py index 1301188..c87918c 100644 --- a/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/converters.py +++ b/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/converters.py @@ -1,4 +1,4 @@ -from fileformats.core import hook +from fileformats.core import converter from fileformats.medimage.base import MedicalImage from fileformats.medimage_mrtrix3 import ( @@ -16,7 +16,7 @@ in_out_file_kwargs = {} -@hook.converter( +@converter( source_format=MedicalImage, target_format=MrtrixImage, out_ext=MrtrixImage.ext, @@ -40,7 +40,7 @@ def mrconvert(name, out_ext: str, **kwargs): return MrConvert(name=name, out_file="out" + out_ext, **kwargs) -@hook.converter( +@converter( source_format=MedicalImage, target_format=MrtrixImageHeader, out_ext=MrtrixImageHeader.ext, diff --git a/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/gradients.py b/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/gradients.py index 4b9f9b5..e25d5da 100644 --- a/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/gradients.py +++ b/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/gradients.py @@ -1,9 +1,10 @@ import numpy as np +from fileformats.core import extra_implementation from fileformats.medimage import DwiEncoding from fileformats.medimage_mrtrix3 import BFile -@DwiEncoding.read_array.register +@extra_implementation(DwiEncoding.read_array) def bfile_read_array(bfile: BFile) -> np.ndarray: return np.asarray( [[float(x) for x in ln.split()] for ln in bfile.read_contents().splitlines()] diff --git a/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/image.py b/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/image.py index 7f58e5b..7ceaaa6 100644 --- a/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/image.py +++ b/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/image.py @@ -4,12 +4,12 @@ from pathlib import Path import numpy as np from medimages4tests.dummy.nifti import get_image as get_dummy_nifti -from fileformats.core import FileSet, SampleFileGenerator +from fileformats.core import FileSet, SampleFileGenerator, extra_implementation from fileformats.medimage import MedicalImage, Nifti1 from fileformats.medimage_mrtrix3 import ImageFormat -@FileSet.generate_sample_data.register +@extra_implementation(FileSet.generate_sample_data) def generate_mrtrix_sample_data( mif: ImageFormat, generator: SampleFileGenerator, @@ -20,7 +20,7 @@ def generate_mrtrix_sample_data( return mif.fspaths -@MedicalImage.read_array.register +@extra_implementation(MedicalImage.read_array) def mrtrix_read_array(mif: ImageFormat) -> np.ndarray: raise NotImplementedError( "Need to work out how to use the metadata to read the array in the correct order" diff --git a/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/tracks.py b/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/tracks.py index a7594b6..b192dc3 100644 --- a/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/tracks.py +++ b/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/tracks.py @@ -3,11 +3,11 @@ import math from pathlib import Path import typing as ty -from fileformats.core import FileSet, SampleFileGenerator +from fileformats.core import FileSet, SampleFileGenerator, extra_implementation from fileformats.medimage_mrtrix3 import Tracks -@FileSet.generate_sample_data.register +@extra_implementation(FileSet.generate_sample_data) def generate_tracks_sample_data( tracks: Tracks, generator: SampleFileGenerator, diff --git a/related-packages/fileformats/fileformats/medimage_mrtrix3/dwi.py b/related-packages/fileformats/fileformats/medimage_mrtrix3/dwi.py index 74bbcff..287e9da 100644 --- a/related-packages/fileformats/fileformats/medimage_mrtrix3/dwi.py +++ b/related-packages/fileformats/fileformats/medimage_mrtrix3/dwi.py @@ -1,4 +1,3 @@ -from fileformats.core import hook from fileformats.core.mixin import WithAdjacentFiles from fileformats.medimage import DwiEncoding, Nifti1, NiftiGz, NiftiX, NiftiGzX from .image import ImageFormat, ImageHeader, ImageFormatGz @@ -12,7 +11,6 @@ class BFile(DwiEncoding): # NIfTI file format gzipped with BIDS side car class WithBFile(WithAdjacentFiles): - @hook.required @property def encoding(self) -> BFile: return BFile(self.select_by_ext(BFile)) @@ -35,12 +33,12 @@ class NiftiGzXB(WithBFile, NiftiGzX): class ImageFormatB(WithBFile, ImageFormat): - pass + iana_mime = "application/x-mrtrix-image-format.b" class ImageFormatGzB(WithBFile, ImageFormatGz): - pass + iana_mime = "application/x-mrtrix-image-format+gzip.b" class ImageHeaderB(WithBFile, ImageHeader): - pass + iana_mime = "application/x-mrtrix-image-header.b" diff --git a/related-packages/fileformats/fileformats/medimage_mrtrix3/image.py b/related-packages/fileformats/fileformats/medimage_mrtrix3/image.py index 1a5c74c..e41f126 100644 --- a/related-packages/fileformats/fileformats/medimage_mrtrix3/image.py +++ b/related-packages/fileformats/fileformats/medimage_mrtrix3/image.py @@ -1,6 +1,4 @@ -import typing as ty from pathlib import Path -from fileformats.core import hook from fileformats.generic import File from fileformats.application import Gzip from fileformats.core.mixin import WithMagicNumber @@ -81,8 +79,9 @@ def dims(self): class ImageFormat(BaseMrtrixImage): ext = ".mif" + iana_mime = "application/x-mrtrix-image-format" - @hook.check + @property def check_data_file(self): if self.data_fspath != self.fspath: raise FormatMismatchError( @@ -97,15 +96,15 @@ def data_file(self): class ImageFormatGz(Gzip[ImageFormat]): - iana_mime = "application/x-image-format-gz" + iana_mime = "application/x-mrtrix-image-format-gz" ext = ".mif.gz" class ImageHeader(BaseMrtrixImage): ext = ".mih" + iana_mime = "application/x-mrtrix-image-header" - @hook.required @property def data_file(self): return ImageDataFile(self.data_fspath) From a3f89ec3f156d37c2368bfdf21ec98d6b5217af2 Mon Sep 17 00:00:00 2001 From: Tom Close Date: Wed, 4 Sep 2024 07:22:17 +1000 Subject: [PATCH 3/9] removed and ignore DS_Store --- .DS_Store | Bin 8196 -> 0 bytes .gitignore | 3 +++ related-packages/.DS_Store | Bin 6148 -> 0 bytes 3 files changed, 3 insertions(+) delete mode 100644 .DS_Store delete mode 100644 related-packages/.DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 540c295c97de9c8b4b76aa0f4a526b528aefaa40..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8196 zcmeHMzl#$=6n>LLj0Pc{*SS?t&Ikc51o74*TKa>4V0CxNhD6Mg==}h#)<)r)1HtnO z3mZYO(f>ffJGHa7xyJSoYz6(^%$R$VWRtTIAv4F!Tjsqt@0)LSX7cukNX@TC6`~m; z%A>L@7SN0+tmisZGPdU=q=G+nT9sO(8~27ReTQejGvFEU40r}S1OE#HxU;$BQ|^7E z^{r>XGcc75@c9s+vJ9Z7sxUR!({xRSt?4*!WuZCa(e6P?a|AGVMfjvqPt_CZ!- z7#Jt+?juRJa_~^dcnZK1(|zipE>MG(P}|gWa9ztjNw<{vHsjL!XQgfCC}muUbF#IA zw-#bpItILqb%mO=j5ZB3B^SqAOWZeaoPPeTtZ|PSUt&L)hl8KKFn(UQH6U+Lm!bil zDfe^OLgM`K>T3Da-4D#~z&H|Hx|NG3z%W>+1^h9i{k7*FXZ_XAR=TCc7tNhr-!kuK z%D58eWNQbn;?0458)4UC{Eu8pu z{k|3vxk(+I3%uhZDgzIn5qHqGcn?ZcnOncU0@XwxiS^CKZt&8as|WDgKjr?$PtGgz zcoz_TnNkL_X5KTL|6fl3{y(Mc_-T6vrh@^MtJbRXn5)gLhi08YPk_3J%7yI`Bee?} pHsOAqjzjkUVTk9z>Ow0MBcVb0=N|&}?+1VXGxz@fXBMWj{08$#q;LQL diff --git a/.gitignore b/.gitignore index c55f867..a0713aa 100644 --- a/.gitignore +++ b/.gitignore @@ -137,4 +137,7 @@ dmypy.json # Hatchling _version.py +# Mac Garbage +.DS_Store + /pydra/tasks/mrtrix3/v3_0 diff --git a/related-packages/.DS_Store b/related-packages/.DS_Store deleted file mode 100644 index 0e140e22d8fea0cff70cc77dae408c61ac06cda6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKF;2r!47J-1K?SK|V$1>P(2*^aup{*XP|^ZYqf`YX#FPs$a0wR9!3CHYdHyzS zqqL|UKnU5A|4aPs#D0n5n25N4pY@13L=-^<8#`E5i0q5jq+u4>FaGy*Q3-|gh8H@GbFsie)_qqp^*d6A~WNil>+^pWj7KQ>>Ezw^$v z@Sa{TWs!zt_E+aiT~zRZH*bdbubTbg(>wCnC~moy*JAgpmJMSUm2Deasq^Fv>AE-r z&VV!UV+^2Yi$rZjU!4JGz!{h`Am4`=Dwr6Sis93NC5!;T7R*6#EVG2f#DOAJdz4JWqYgYC|26AFjB<$4y*7rPLq)`|RGf$4AhcrmN-I8q27x`617KoUD#8Ns OAAvxFFV4WPGVlqoS!i(p From 51156c5d9617ab3177b93cd8b0491b7ec64d909b Mon Sep 17 00:00:00 2001 From: Tom Close Date: Wed, 4 Sep 2024 10:52:06 +1000 Subject: [PATCH 4/9] fixed up changes to syntax to match core --- .../fileformats/extras/medimage_mrtrix3/image.py | 2 +- .../fileformats/extras/medimage_mrtrix3/tracks.py | 2 +- .../fileformats/fileformats/medimage_mrtrix3/image.py | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/image.py b/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/image.py index 7ceaaa6..be543ee 100644 --- a/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/image.py +++ b/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/image.py @@ -13,7 +13,7 @@ def generate_mrtrix_sample_data( mif: ImageFormat, generator: SampleFileGenerator, -) -> ty.Iterable[Path]: +) -> ty.List[Path]: nifti = Nifti1(get_dummy_nifti(generator.dest_dir / "nifti.nii")) with mock.patch.dict(os.environ, {"MRTRIX_CLI_PARSE_ONLY": "0"}): mif = ImageFormat.convert(nifti) diff --git a/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/tracks.py b/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/tracks.py index b192dc3..2ef9f37 100644 --- a/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/tracks.py +++ b/related-packages/fileformats-extras/fileformats/extras/medimage_mrtrix3/tracks.py @@ -11,7 +11,7 @@ def generate_tracks_sample_data( tracks: Tracks, generator: SampleFileGenerator, -) -> ty.Iterable[Path]: +) -> ty.List[Path]: """Generate a tracks file with a single straight track of length 10""" fspath = generator.dest_dir / "tracks.tck" timestamp = str(time.time() * 1e9 + time.process_time_ns()) diff --git a/related-packages/fileformats/fileformats/medimage_mrtrix3/image.py b/related-packages/fileformats/fileformats/medimage_mrtrix3/image.py index e41f126..8d1638e 100644 --- a/related-packages/fileformats/fileformats/medimage_mrtrix3/image.py +++ b/related-packages/fileformats/fileformats/medimage_mrtrix3/image.py @@ -65,7 +65,9 @@ def data_fspath(self): @property def data_offset(self): - return int(self.metadata["file"].split()[1]) + fspath_and_offset = self.metadata["file"].split() + assert len(fspath_and_offset) <= 2 + return int(fspath_and_offset[1]) if len(fspath_and_offset) > 1 else 0 @property def vox_sizes(self): From 3e941588da8d971c8555a8817de1bdc0c9776eb2 Mon Sep 17 00:00:00 2001 From: Tom Close Date: Wed, 4 Sep 2024 22:20:15 +1000 Subject: [PATCH 5/9] moved read_metadata into extra_implementation method --- .../fileformats/medimage_mrtrix3/image.py | 82 ++++++++++--------- 1 file changed, 45 insertions(+), 37 deletions(-) diff --git a/related-packages/fileformats/fileformats/medimage_mrtrix3/image.py b/related-packages/fileformats/fileformats/medimage_mrtrix3/image.py index 8d1638e..26bec5a 100644 --- a/related-packages/fileformats/fileformats/medimage_mrtrix3/image.py +++ b/related-packages/fileformats/fileformats/medimage_mrtrix3/image.py @@ -1,4 +1,6 @@ from pathlib import Path +import typing as ty +from fileformats.core import FileSet, extra_implementation from fileformats.generic import File from fileformats.application import Gzip from fileformats.core.mixin import WithMagicNumber @@ -15,43 +17,6 @@ class BaseMrtrixImage(WithMagicNumber, fileformats.medimage.MedicalImage, File): magic_number = b"mrtrix image\n" binary = True - def read_metadata(self): - metadata = {} - with open(self.fspath, "rb") as f: - line = f.readline() - if line != self.magic_number: - raise FormatMismatchError( - f"Magic line {line} doesn't match reference {self.magic_number}" - ) - line = f.readline().decode("utf-8") - while line and line != "END\n": - key, value = line.split(": ", maxsplit=1) - if "," in value: - try: - value = [int(v) for v in value.split(",")] - except ValueError: - try: - value = [float(v) for v in value.split(",")] - except ValueError: - pass - else: - try: - value = int(value) - except ValueError: - try: - value = float(value) - except ValueError: - pass - if key in metadata: - if isinstance(metadata[key], MultiLineMetadataValue): - metadata[key].append(value) - else: - metadata[key] = MultiLineMetadataValue([metadata[key], value]) - else: - metadata[key] = value - line = f.readline().decode("utf-8") - return metadata - @property def data_fspath(self): data_fspath = self.metadata["file"].split()[0] @@ -121,3 +86,46 @@ def __attrs_post_init__(self): class ImageDataFile(File): ext = ".dat" + + +@extra_implementation(FileSet.read_metadata) +def mrtrix_read_metadata( + mif: BaseMrtrixImage, selected_keys: ty.Optional[ty.Sequence[str]] = None +) -> ty.Mapping[str, ty.Any]: + metadata = {} + with open(mif.fspath, "rb") as f: + line = f.readline() + if line != mif.magic_number: + raise FormatMismatchError( + f"Magic line {line} doesn't match reference {mif.magic_number}" + ) + line = f.readline().decode("utf-8") + while line and line != "END\n": + key, value = line.split(": ", maxsplit=1) + if "," in value: + try: + value = [int(v) for v in value.split(",")] + except ValueError: + try: + value = [float(v) for v in value.split(",")] + except ValueError: + pass + else: + try: + value = int(value) + except ValueError: + try: + value = float(value) + except ValueError: + pass + if key in metadata: + if isinstance(metadata[key], MultiLineMetadataValue): + metadata[key].append(value) + else: + metadata[key] = MultiLineMetadataValue([metadata[key], value]) + else: + metadata[key] = value + line = f.readline().decode("utf-8") + if selected_keys: + metadata = {k: v for k, v in metadata.items() if k in selected_keys} + return metadata From e03104d57c3b7425ae9eb86fe7f34fddd64c044c Mon Sep 17 00:00:00 2001 From: Tom Close Date: Thu, 5 Sep 2024 06:43:14 +1000 Subject: [PATCH 6/9] update to match read_metadata signature --- .../fileformats/fileformats/medimage_mrtrix3/image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/related-packages/fileformats/fileformats/medimage_mrtrix3/image.py b/related-packages/fileformats/fileformats/medimage_mrtrix3/image.py index 26bec5a..1b4b587 100644 --- a/related-packages/fileformats/fileformats/medimage_mrtrix3/image.py +++ b/related-packages/fileformats/fileformats/medimage_mrtrix3/image.py @@ -90,7 +90,7 @@ class ImageDataFile(File): @extra_implementation(FileSet.read_metadata) def mrtrix_read_metadata( - mif: BaseMrtrixImage, selected_keys: ty.Optional[ty.Sequence[str]] = None + mif: BaseMrtrixImage, selected_keys: ty.Optional[ty.Collection[str]] = None ) -> ty.Mapping[str, ty.Any]: metadata = {} with open(mif.fspath, "rb") as f: From 9db3f2d6fc6ce62c538044df80d0b80b789936b0 Mon Sep 17 00:00:00 2001 From: Tom Close Date: Tue, 10 Sep 2024 11:40:36 +1000 Subject: [PATCH 7/9] updated ci to pull in github tags for python versioning --- .github/workflows/ci-cd.yml | 79 +++---------------------------------- pyproject.toml | 5 +-- 2 files changed, 8 insertions(+), 76 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 945c2b6..3506aef 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -45,23 +45,18 @@ jobs: run: | git config --global user.email "dummy@email.com" git config --global user.name "Dummy User" - - name: install dependencies run: | sudo apt-get update sudo apt-get install clang qt6-base-dev libglvnd-dev libeigen3-dev zlib1g-dev libfftw3-dev ninja-build - - name: Run sccache-cache uses: mozilla-actions/sccache-action@v0.0.3 - - name: Get CMake uses: lukka/get-cmake@latest with: cmakeVersion: '3.16.3' - - name: Print CMake version run: cmake --version - - name: Clone latest MRtrix and switch to latest tag run: | mkdir $MRTRIX_HOME @@ -71,10 +66,8 @@ jobs: git checkout pydra-usage-python-cmd git tag -a $MRTRIX_VERSION -m"Tag used to create a pydra-mrtrix3 release" git describe --abbrev=0 - # echo "MRTRIX_VERSION=$(git describe --tags --abbrev=0)" >> $GITHUB_ENV # git checkout $MRTRIX_VERSION - - name: configure run: | cd $MRTRIX_HOME/src @@ -88,40 +81,31 @@ jobs: -D CMAKE_C_COMPILER=clang \ -D CMAKE_CXX_COMPILER=clang++ \ -D CMAKE_INSTALL_PREFIX=$MRTRIX_INSTALL - - name: Build Mrtrix run: | cd $MRTRIX_HOME/src cmake --build build - - name: Install Mrtrix run: | cd $MRTRIX_HOME/src cmake --install build - - name: Set PATH Variable run: echo "PATH=$PATH:$MRTRIX_INSTALL/bin" >> $GITHUB_ENV - - name: Set LD_LIBRARY_PATH Variable run: echo "LD_LIBRARY_PATH=$MRTRIX_INSTALL/lib" >> $GITHUB_ENV - - name: Change back to the root directory run: cd .. - - name: Set up Python uses: actions/setup-python@v2 - - name: Install Python build dependencies run: | python -m pip install --upgrade pip - - name: Install pydra-auto-gen requirements run: > pip install -e related-packages/fileformats -e related-packages/fileformats-extras -e .[dev,test] - - name: Generate task specifications run: > ./generate.py @@ -130,22 +114,18 @@ jobs: $MRTRIX_VERSION --log-errors --latest - - name: Upload MRtrix3 install uses: actions/upload-artifact@v2 with: name: MRtrix3 path: ${{ env.MRTRIX_INSTALL}} - - name: Upload auto-gen pydra uses: actions/upload-artifact@v2 with: name: AutoGen path: pydra/tasks/mrtrix3/${{ env.SUBPKG_NAME }} - - name: Write version file run: echo $MRTRIX_VERSION > mrtrix3_version.txt - - name: Upload version file uses: actions/upload-artifact@v2 with: @@ -165,40 +145,32 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Download version file uses: actions/download-artifact@v2 with: name: VersionFile path: mrtrix3_version.txt - - name: Extract Mrtrix version run: echo "MRTRIX_VERSION=$(cat mrtrix3_version.txt)" >> $GITHUB_ENV - - name: Download auto-gen pydra uses: actions/download-artifact@v2 with: name: AutoGen path: pydra/tasks/mrtrix3/${{ env.SUBPKG_NAME }} - - name: Strip auto package from gitignore so it is included in package run: | sed -i '/\/pydra\/tasks\/mrtrix3\/${{ env.SUBPKG_NAME }}/d' .gitignore - - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Install build dependencies run: | python -m pip install --upgrade pip - - name: Install Pydra run: | pip install ${{ matrix.pydra }} python -c "import pydra as m; print(f'{m.__name__} {m.__version__} @ {m.__file__}')" - - name: Install task package run: | pip install ${{ matrix.pip-flags }} "related-packages/fileformats" @@ -221,66 +193,57 @@ jobs: run: | sudo apt-get update sudo apt-get install clang qt6-base-dev libglvnd-dev libeigen3-dev zlib1g-dev libfftw3-dev ninja-build - - name: Checkout code uses: actions/checkout@v2 - + - name: Unset header + # checkout@v2 adds a header that makes branch protection report errors + # because the Github action bot is not a collaborator on the repo + run: git config --local --unset http.https://github.com/.extraheader + - name: Fetch tags + run: git fetch --prune --unshallow - name: Download version file uses: actions/download-artifact@v2 with: name: VersionFile path: mrtrix3_version.txt - - name: Extract Mrtrix version run: echo "MRTRIX_VERSION=$(cat mrtrix3_version.txt)" >> $GITHUB_ENV - - name: Download auto-gen pydra uses: actions/download-artifact@v2 with: name: AutoGen path: pydra/tasks/mrtrix3/${{ env.SUBPKG_NAME }} - - name: Strip auto package from gitignore so it is included in package run: | sed -i '/\/pydra\/tasks\/mrtrix3\/${{ env.SUBPKG_NAME }}/d' .gitignore - - name: Download MRtrix3 install uses: actions/download-artifact@v2 with: name: MRtrix3 path: ${{ env.MRTRIX_INSTALL}} - - name: Make commands executable run: chmod +x ${{ env.MRTRIX_INSTALL }}/bin/* - - name: Set PATH Variable run: echo "PATH=$PATH:$MRTRIX_INSTALL/bin" >> $GITHUB_ENV - - name: Set LD_LIBRARY_PATH Variable run: echo "LD_LIBRARY_PATH=$MRTRIX_INSTALL/lib" >> $GITHUB_ENV - - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Install build dependencies run: | python -m pip install --upgrade pip - - name: Install task package run: > pip install -e ./related-packages/fileformats -e ./related-packages/fileformats-extras -e '.[test]' - - name: Install dev branch of pydra until bugfix run: pip install --upgrade git+https://github.com/nipype/pydra.git@typing-bugfixes - - name: Test with pytest run: pytest -sv pydra/tasks/mrtrix3 --cov pydra.tasks.mrtrix3 --cov-report xml - - uses: codecov/codecov-action@v1 if: ${{ always() }} @@ -288,33 +251,26 @@ jobs: needs: [devcheck, test] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 with: submodules: recursive fetch-depth: 0 - - name: Set up Python uses: actions/setup-python@v4 with: python-version: '3.11' - - name: Install build tools run: python -m pip install build twine - - name: Build source and wheel distributions run: python -m build ./related-packages/fileformats - - name: Check distributions run: twine check ./related-packages/fileformats/dist/* - - name: Check for PyPI token on tag id: deployable if: github.event_name == 'release' || github.event_name == 'repository_dispatch' env: PYPI_API_TOKEN: "${{ secrets.PYPI_FILEFORMATS_API_TOKEN }}" run: if [ -n "$PYPI_API_TOKEN" ]; then echo "DEPLOY=true" >> $GITHUB_OUTPUT; fi - - name: Upload to PyPI if: steps.deployable.outputs.DEPLOY uses: pypa/gh-action-pypi-publish@release/v1 @@ -327,33 +283,26 @@ jobs: needs: [deploy-fileformats] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 with: submodules: recursive fetch-depth: 0 - - name: Set up Python uses: actions/setup-python@v4 with: python-version: '3.11' - - name: Install build tools run: python -m pip install build twine - - name: Build source and wheel distributions run: python -m build ./related-packages/fileformats-extras - - name: Check distributions run: twine check ./related-packages/fileformats-extras/dist/* - - name: Check for PyPI token on tag id: deployable if: github.event_name == 'release' || github.event_name == 'repository_dispatch' env: PYPI_API_TOKEN: "${{ secrets.PYPI_FILEFORMATS_EXTRAS_API_TOKEN }}" run: if [ -n "$PYPI_API_TOKEN" ]; then echo "DEPLOY=true" >> $GITHUB_OUTPUT; fi - - name: Upload to PyPI if: steps.deployable.outputs.DEPLOY uses: pypa/gh-action-pypi-publish@release/v1 @@ -367,73 +316,58 @@ jobs: needs: [devcheck, test, deploy-fileformats, deploy-fileformats-extras] runs-on: ubuntu-latest steps: - - name: Checkout repo uses: actions/checkout@v2 - - name: Set up Git user run: | git config --local user.email "action@github.com" git config --local user.name "GitHub Action" - - name: Download version file uses: actions/download-artifact@v2 with: name: VersionFile path: mrtrix3_version.txt - - name: Extract Mrtrix version run: echo "MRTRIX_VERSION=$(cat mrtrix3_version.txt)" >> $GITHUB_ENV - - name: Download auto-gen pydra uses: actions/download-artifact@v2 with: name: AutoGen path: pydra/tasks/mrtrix3/${{ env.SUBPKG_NAME }} - - name: Strip auto package from gitignore so it is included in package run: | sed -i '/\/pydra\/tasks\/mrtrix3\/${{ env.SUBPKG_NAME }}/d' .gitignore - - name: Add auto-generated directory to git repo if: github.event_name == 'release' || github.event_name == 'repository_dispatch' run: | git add pydra/tasks/mrtrix3/$SUBPKG_NAME git commit -am"added auto-generated version to make new tag for package version" git status - - name: Get latest version tag id: latest_tag run: | git fetch --tags echo "TAG=$(git tag -l | grep 'v.*' | tail -n 1 | awk -F post '{print $1}')" >> $GITHUB_OUTPUT - - name: Overwrite the tag of release event with latest commit (i.e. including the auto directory) if: github.event_name == 'release' run: | git tag -d ${{ steps.latest_tag.outputs.TAG }}; git tag -a ${{ steps.latest_tag.outputs.TAG }} -m"Tag used to create a pydra-mrtrix3 $MRTRIX_VERSION release"; - - name: Set up Python 3.11 uses: actions/setup-python@v2 with: python-version: 3.11 - - name: Install build tools run: python -m pip install --upgrade pip twine build - - name: Build source and wheel distributions run: python -m build - - name: Check distributions run: twine check dist/* - - name: Upload sdist uses: actions/upload-artifact@v2 with: name: SDist path: dist/*.tar.gz - # Deploy on tags if PYPI_API_TOKEN is defined in the repository secrets. # Secrets are not accessible in the if: condition [0], so set an output variable [1] # [0] https://github.community/t/16928 @@ -444,7 +378,6 @@ jobs: env: PYPI_API_TOKEN: "${{ secrets.PYPI_API_TOKEN }}" run: if [ -n "$PYPI_API_TOKEN" ]; then echo ::set-output name=DEPLOY::true; fi - - name: Upload to PyPI if: steps.deployable.outputs.DEPLOY uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/pyproject.toml b/pyproject.toml index ad2fb37..8b5799a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ description = "pydra-mrtrix3 contains Pydra task specifications for MRtrix3 tool readme = "README.rst" requires-python = ">=3.8" dependencies = [ - "fileformats-medimage_mrtrix3 >=3.0.4a4", + "fileformats-medimage_mrtrix3 >=3.0.4a7", "numpy", "pydra >=0.23", ] @@ -36,8 +36,7 @@ dev = [ "click >=8.1.3", "tqdm", "attrs >=23.1.0", - "fileformats >= 0.8", - "fileformats-extras >= 0.2.1", + "fileformats-extras >= 0.12.1", ] doc = [ "packaging", From 1bbfcc123d92b0e2136390a28225f8e2c57db7ec Mon Sep 17 00:00:00 2001 From: Tom Close Date: Tue, 10 Sep 2024 12:06:19 +1000 Subject: [PATCH 8/9] relaxed fileformats version requirements --- pyproject.toml | 2 +- related-packages/fileformats-extras/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8b5799a..23d4949 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ description = "pydra-mrtrix3 contains Pydra task specifications for MRtrix3 tool readme = "README.rst" requires-python = ">=3.8" dependencies = [ - "fileformats-medimage_mrtrix3 >=3.0.4a7", + "fileformats-medimage_mrtrix3 >=3.0.4a5", "numpy", "pydra >=0.23", ] diff --git a/related-packages/fileformats-extras/pyproject.toml b/related-packages/fileformats-extras/pyproject.toml index 7155dad..0ee7929 100644 --- a/related-packages/fileformats-extras/pyproject.toml +++ b/related-packages/fileformats-extras/pyproject.toml @@ -9,7 +9,7 @@ readme = "README.rst" requires-python = ">=3.8" dependencies = [ "fileformats >= 0.8", - "fileformats-medimage-mrtrix3 >=3.0.4a4", + "fileformats-medimage-mrtrix3 >=3.0.4a5", "medimages4tests", "pydra >= 0.23.0", ] From 3b86ac9d21eb7add316ef92dcec9bdc8f9b17c1f Mon Sep 17 00:00:00 2001 From: Tom Close Date: Tue, 10 Sep 2024 12:18:25 +1000 Subject: [PATCH 9/9] disabled pydra branch install in ci --- .github/workflows/ci-cd.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 3506aef..c8405bc 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -240,8 +240,6 @@ jobs: -e ./related-packages/fileformats -e ./related-packages/fileformats-extras -e '.[test]' - - name: Install dev branch of pydra until bugfix - run: pip install --upgrade git+https://github.com/nipype/pydra.git@typing-bugfixes - name: Test with pytest run: pytest -sv pydra/tasks/mrtrix3 --cov pydra.tasks.mrtrix3 --cov-report xml - uses: codecov/codecov-action@v1