From 39c9c9334943f74136657b31068e0f539b7c395a Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Thu, 14 Dec 2023 15:20:03 +0100 Subject: [PATCH 1/9] use logger --- giga_connectome/atlas.py | 8 +++- giga_connectome/logger.py | 16 ++++++++ giga_connectome/mask.py | 60 ++++++++++++++---------------- giga_connectome/postprocess.py | 15 ++++++-- giga_connectome/run.py | 11 ++++++ giga_connectome/tests/test_mask.py | 3 +- giga_connectome/utils.py | 11 ++++-- giga_connectome/workflow.py | 40 ++++++++++++++++---- tools/download_templates.py | 8 +++- 9 files changed, 120 insertions(+), 52 deletions(-) create mode 100644 giga_connectome/logger.py diff --git a/giga_connectome/atlas.py b/giga_connectome/atlas.py index d3e91d2..d0c7515 100644 --- a/giga_connectome/atlas.py +++ b/giga_connectome/atlas.py @@ -9,6 +9,10 @@ from nibabel import Nifti1Image from pkg_resources import resource_filename +from giga_connectome.logger import gc_logger + +gc_log = gc_logger() + PRESET_ATLAS = ["DiFuMo", "MIST", "Schaefer20187Networks"] @@ -42,7 +46,7 @@ def load_atlas_setting(atlas: Union[str, Path, dict]): Path to the atlas files. """ atlas_config = _check_altas_config(atlas) - print(atlas_config) + gc_log.info(atlas_config) # load template flow templateflow_dir = atlas_config.get("templateflow_dir") @@ -104,7 +108,7 @@ def resample_atlas_collection( List of pathlib.Path Paths to atlases sampled to group level grey matter mask. """ - print("Resample atlas to group grey matter mask.") + gc_log.info("Resample atlas to group grey matter mask.") resampled_atlases = [] for desc in tqdm(atlas_config["file_paths"]): parcellation = atlas_config["file_paths"][desc] diff --git a/giga_connectome/logger.py b/giga_connectome/logger.py new file mode 100644 index 0000000..c0c69ec --- /dev/null +++ b/giga_connectome/logger.py @@ -0,0 +1,16 @@ +"""General logger for the cohort_creator package.""" +from __future__ import annotations + +import logging + + +def gc_logger(log_level: str = "INFO") -> logging.Logger: + FORMAT = "%(message)s" + + logging.basicConfig( + level=log_level, + format=FORMAT, + datefmt="[%X]", + ) + + return logging.getLogger("giga_connectome") diff --git a/giga_connectome/mask.py b/giga_connectome/mask.py index 80d9d55..7c9a4f1 100644 --- a/giga_connectome/mask.py +++ b/giga_connectome/mask.py @@ -19,6 +19,10 @@ from giga_connectome.atlas import resample_atlas_collection +from giga_connectome.logger import gc_logger + +gc_log = gc_logger() + def generate_gm_mask_atlas( working_dir: Path, @@ -63,7 +67,6 @@ def generate_group_mask( template: str = "MNI152NLin2009cAsym", templateflow_dir: Optional[Path] = None, n_iter: int = 2, - verbose: int = 1, ) -> Nifti1Image: """ Generate a group EPI grey matter mask, and overlaid with a MNI grey @@ -88,9 +91,6 @@ def generate_group_mask( Number of repetitions of dilation and erosion steps performed in scipy.ndimage.binary_closing function. - verbose : - Level of verbosity. - Keyword Arguments ----------------- Used to filter the cirret @@ -102,12 +102,10 @@ def generate_group_mask( nibabel.nifti1.Nifti1Image EPI (grey matter) mask for the current group of subjects. """ - if verbose > 1: - print(f"Found {len(imgs)} masks") - if exclude := _check_mask_affine(imgs, verbose): + gc_log.debug(f"Found {len(imgs)} masks") + if exclude := _check_mask_affine(imgs): imgs, __annotations__ = _get_consistent_masks(imgs, exclude) - if verbose > 1: - print(f"Remaining: {len(imgs)} masks") + gc_log.debug(f"Remaining: {len(imgs)} masks") # templateflow environment setting to get around network issue if templateflow_dir and templateflow_dir.exists(): @@ -129,7 +127,7 @@ def generate_group_mask( memory=None, verbose=0, ) - print( + gc_log.info( f"Group EPI mask affine:\n{group_epi_mask.affine}" f"\nshape: {group_epi_mask.shape}" ) @@ -204,7 +202,7 @@ def _get_consistent_masks( def _check_mask_affine( - mask_imgs: List[Union[Path, str, Nifti1Image]], verbose: int = 1 + mask_imgs: List[Union[Path, str, Nifti1Image]] ) -> Union[list, None]: """Given a list of input mask images, show the most common affine matrix and subjects with different values. @@ -215,9 +213,6 @@ def _check_mask_affine( See :ref:`extracting_data`. 3D or 4D EPI image with same affine. - verbose : - Level of verbosity. - Returns ------- @@ -244,12 +239,11 @@ def _check_mask_affine( common_affine = max( set(header_info["affine"]), key=header_info["affine"].count ) - if verbose > 0: - print( - f"We found {len(set(header_info['affine']))} unique affine " - f"matrices. The most common one is " - f"{key_to_header[common_affine]}" - ) + gc_log.debug( + f"We found {len(set(header_info['affine']))} unique affine " + f"matrices. The most common one is " + f"{key_to_header[common_affine]}" + ) odd_balls = set(header_info["affine"]) - {common_affine} if not odd_balls: return None @@ -259,18 +253,16 @@ def _check_mask_affine( ob_index = [ i for i, aff in enumerate(header_info["affine"]) if aff == ob ] - if verbose > 1: - print( - "The following subjects has a different affine matrix " - f"({key_to_header[ob]}) comparing to the most common value: " - f"{mask_imgs[ob_index]}." - ) - exclude += ob_index - if verbose > 0: - print( - f"{len(exclude)} out of {len(mask_imgs)} has " - "different affine matrix. Ignore when creating group mask." + gc_log.debug( + "The following subjects has a different affine matrix " + f"({key_to_header[ob]}) comparing to the most common value: " + f"{mask_imgs[ob_index]}." ) + exclude += ob_index + gc_log.info( + f"{len(exclude)} out of {len(mask_imgs)} has " + "different affine matrix. Ignore when creating group mask." + ) return sorted(exclude) @@ -284,7 +276,9 @@ def _check_pregenerated_masks(template, working_dir, atlas): if not group_mask.exists(): group_mask = None else: - print(f"Found pregenerated group level grey matter mask: {group_mask}") + gc_log.info( + f"Found pregenerated group level grey matter mask: {group_mask}" + ) # atlas resampled_atlases = [] @@ -301,7 +295,7 @@ def _check_pregenerated_masks(template, working_dir, atlas): if not all(all_exist): resampled_atlases = None else: - print( + gc_log.info( f"Found resampled atlases: {resampled_atlases}. Skipping group " "level mask generation step." ) diff --git a/giga_connectome/postprocess.py b/giga_connectome/postprocess.py index 6cf4d8a..3187dca 100644 --- a/giga_connectome/postprocess.py +++ b/giga_connectome/postprocess.py @@ -7,9 +7,13 @@ from nilearn.connectome import ConnectivityMeasure from nilearn.maskers import NiftiLabelsMasker, NiftiMapsMasker from bids.layout import BIDSImageFile + from giga_connectome import utils from giga_connectome.connectome import generate_timeseries_connectomes from giga_connectome.denoise import denoise_nifti_voxel +from giga_connectome.logger import gc_logger + +gc_log = gc_logger() def run_postprocessing_dataset( @@ -97,7 +101,8 @@ def run_postprocessing_dataset( ) # transform data - print("processing subjects") + gc_log.info("processing subjects") + for img in tqdm(images): # process timeseries denoised_img = denoise_nifti_voxel( @@ -109,7 +114,9 @@ def run_postprocessing_dataset( attribute_name = f"{subject}_{specifier}_atlas-{atlas}_desc-{desc}" if not denoised_img: time_series_atlas, correlation_matrix = None, None - print(f"{attribute_name}: no volume after scrubbing") + + gc_log.info(f"{attribute_name}: no volume after scrubbing") + continue # extract timeseries and connectomes @@ -140,7 +147,9 @@ def run_postprocessing_dataset( ) if analysis_level == "group": - print("create group connectome") + + gc_log.info("create group connectome") + for desc in connectomes: average_connectome = np.mean( np.array(connectomes[desc]), axis=0 diff --git a/giga_connectome/run.py b/giga_connectome/run.py index b39365d..2d93bba 100644 --- a/giga_connectome/run.py +++ b/giga_connectome/run.py @@ -105,6 +105,17 @@ def main(argv=None): "pipeline (option A). The default is False.", action="store_true", ) + parser.add_argument( + "--verbosity", + help=""" + Verbosity level. + """, + required=False, + choices=[0, 1, 2, 3], + default=2, + type=int, + nargs=1, + ) args = parser.parse_args(argv) diff --git a/giga_connectome/tests/test_mask.py b/giga_connectome/tests/test_mask.py index 0369bc1..9afac35 100644 --- a/giga_connectome/tests/test_mask.py +++ b/giga_connectome/tests/test_mask.py @@ -33,8 +33,7 @@ def test_check_mask_affine(): weird = Nifti1Image(processed_vol, np.eye(4) * np.array([1, 1, 1.5, 1]).T) weird2 = Nifti1Image(processed_vol, np.eye(4) * np.array([1, 1, 1.6, 1]).T) exclude = mask._check_mask_affine( - [processed, processed, processed, processed, weird, weird, weird2], - verbose=2, + [processed, processed, processed, processed, weird, weird, weird2] ) assert len(exclude) == 3 assert exclude == [4, 5, 6] diff --git a/giga_connectome/utils.py b/giga_connectome/utils.py index cc5f801..283db50 100644 --- a/giga_connectome/utils.py +++ b/giga_connectome/utils.py @@ -4,6 +4,10 @@ from bids.layout import Query from bids import BIDSLayout +from giga_connectome.logger import gc_logger + +gc_log = gc_logger() + def get_bids_images( subjects: List[str], @@ -175,7 +179,7 @@ def get_subject_lists( ] -def check_path(path: Path, verbose=True): +def check_path(path: Path): """Check if given path (file or dir) already exists, and if so returns a new path with _ appended (n being the number of paths with the same name that exist already). @@ -196,6 +200,7 @@ def check_path(path: Path, verbose=True): ] n = str(max(existing_numbers) + 1) if existing_numbers else "1" path = path_parent / f"{path.stem}_{n}{ext}" - if verbose: - print(f"Specified path already exists, using {path} instead.") + + gc_log.debug(f"Specified path already exists, using {path} instead.") + return path diff --git a/giga_connectome/workflow.py b/giga_connectome/workflow.py index 4f1a59d..5a9a1f5 100644 --- a/giga_connectome/workflow.py +++ b/giga_connectome/workflow.py @@ -10,10 +10,29 @@ from giga_connectome.denoise import is_ica_aroma from giga_connectome import utils +from giga_connectome.logger import gc_logger + + +gc_log = gc_logger() + + +def set_verbosity(verbosity: int | list[int]) -> None: + if isinstance(verbosity, list): + verbosity = verbosity[0] + if verbosity == 0: + gc_log.setLevel("ERROR") + elif verbosity == 1: + gc_log.setLevel("WARNING") + elif verbosity == 2: + gc_log.setLevel("INFO") + elif verbosity == 3: + gc_log.setLevel("DEBUG") def workflow(args): - print(vars(args)) + + gc_log.info(vars(args)) + # set file paths bids_dir = args.bids_dir output_dir = args.output_dir @@ -30,6 +49,8 @@ def workflow(args): strategy = get_denoise_strategy(args.denoise_strategy) atlas = load_atlas_setting(args.atlas) + set_verbosity(args.verbosity) + # check output path output_dir.mkdir(parents=True, exist_ok=True) working_dir.mkdir(parents=True, exist_ok=True) @@ -38,7 +59,8 @@ def workflow(args): template = ( "MNI152NLin6Asym" if is_ica_aroma(strategy) else "MNI152NLin2009cAsym" ) - print("Indexing BIDS directory") + + gc_log.info("Indexing BIDS directory") # create subject ts and connectomes # refactor the two cases into one @@ -55,8 +77,10 @@ def workflow(args): f"sub-{subject}_atlas-{atlas['name']}" f"_desc-{strategy['name']}.h5" ) - connectome_path = utils.check_path(connectome_path, verbose=True) - print("Generate subject level connectomes") + connectome_path = utils.check_path(connectome_path) + + gc_log.info("Generate subject level connectomes") + run_postprocessing_dataset( strategy, resampled_atlases, @@ -80,9 +104,11 @@ def workflow(args): connectome_path = ( output_dir / f"atlas-{atlas['name']}_desc-{strategy['name']}.h5" ) - connectome_path = utils.check_path(connectome_path, verbose=True) - print(connectome_path) - print("Generate subject level connectomes") + connectome_path = utils.check_path(connectome_path) + + gc_log.info(connectome_path) + gc_log.info("Generate subject level connectomes") + run_postprocessing_dataset( strategy, resampled_atlases, diff --git a/tools/download_templates.py b/tools/download_templates.py index c350086..d79a421 100644 --- a/tools/download_templates.py +++ b/tools/download_templates.py @@ -4,6 +4,10 @@ """ from pathlib import Path +from giga_connectome.logger import gc_logger + +gc_log = gc_logger() + def fetch_tpl_atlas(): """Download datasets from templateflow.""" @@ -13,7 +17,7 @@ def fetch_tpl_atlas(): for atlas in atlases: tf_path = tf.get("MNI152NLin2009cAsym", atlas=atlas) if isinstance(tf_path, list) and len(tf_path) > 0: - print(f"{atlas} exists.") + gc_log.info(f"{atlas} exists.") # download MNI grey matter template tf.get("MNI152NLin2009cAsym", label="GM") @@ -24,7 +28,7 @@ def download_mist(): tf_path = templateflow.api.get("MNI152NLin2009bAsym", atlas="BASC") if isinstance(tf_path, list) and len(tf_path) > 0: - print("BASC / MIST atlas exists.") + gc_log.info("BASC / MIST atlas exists.") return # download and convert From b152fa5cb84d6e62015e57dd68928c74a9acced9 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Thu, 14 Dec 2023 15:24:50 +0100 Subject: [PATCH 2/9] Update giga_connectome/mask.py --- giga_connectome/mask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/giga_connectome/mask.py b/giga_connectome/mask.py index 7c9a4f1..8e42bc7 100644 --- a/giga_connectome/mask.py +++ b/giga_connectome/mask.py @@ -239,7 +239,7 @@ def _check_mask_affine( common_affine = max( set(header_info["affine"]), key=header_info["affine"].count ) - gc_log.debug( + gc_log.info( f"We found {len(set(header_info['affine']))} unique affine " f"matrices. The most common one is " f"{key_to_header[common_affine]}" From a57c4f705bc588a607e8ade92af5a31786894b13 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Thu, 14 Dec 2023 15:29:26 +0100 Subject: [PATCH 3/9] add future annotations --- giga_connectome/workflow.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/giga_connectome/workflow.py b/giga_connectome/workflow.py index 5a9a1f5..1d19e26 100644 --- a/giga_connectome/workflow.py +++ b/giga_connectome/workflow.py @@ -1,6 +1,8 @@ """ Process fMRIPrep outputs to timeseries based on denoising strategy. """ +from __future__ import annotations + from giga_connectome import ( generate_gm_mask_atlas, load_atlas_setting, From e517cfefbd072cfa0c570440e50f85556bd6b6f7 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Thu, 14 Dec 2023 16:22:32 +0100 Subject: [PATCH 4/9] use rich --- .gitignore | 1 + giga_connectome/logger.py | 4 ++++ giga_connectome/mask.py | 4 ++-- giga_connectome/postprocess.py | 10 ++++++++-- giga_connectome/workflow.py | 4 ++-- pyproject.toml | 1 + 6 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index e0e260c..04538e5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ */_version.py */data/test_data/ output/ +work/ # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/giga_connectome/logger.py b/giga_connectome/logger.py index c0c69ec..e859451 100644 --- a/giga_connectome/logger.py +++ b/giga_connectome/logger.py @@ -3,14 +3,18 @@ import logging +from rich.logging import RichHandler + def gc_logger(log_level: str = "INFO") -> logging.Logger: + # FORMAT = '\n%(asctime)s - %(name)s - %(levelname)s\n\t%(message)s\n' FORMAT = "%(message)s" logging.basicConfig( level=log_level, format=FORMAT, datefmt="[%X]", + handlers=[RichHandler()], ) return logging.getLogger("giga_connectome") diff --git a/giga_connectome/mask.py b/giga_connectome/mask.py index 8e42bc7..27253cc 100644 --- a/giga_connectome/mask.py +++ b/giga_connectome/mask.py @@ -296,7 +296,7 @@ def _check_pregenerated_masks(template, working_dir, atlas): resampled_atlases = None else: gc_log.info( - f"Found resampled atlases: {resampled_atlases}. Skipping group " - "level mask generation step." + f"Found resampled atlases:\n{[str(x) for x in resampled_atlases]}." + "\nSkipping group level mask generation step." ) return group_mask, resampled_atlases diff --git a/giga_connectome/postprocess.py b/giga_connectome/postprocess.py index 3187dca..20a2564 100644 --- a/giga_connectome/postprocess.py +++ b/giga_connectome/postprocess.py @@ -101,9 +101,13 @@ def run_postprocessing_dataset( ) # transform data - gc_log.info("processing subjects") + gc_log.info("Processing subject") for img in tqdm(images): + + print() + gc_log.info(f"Processing image:\n{img.filename}") + # process timeseries denoised_img = denoise_nifti_voxel( strategy, group_mask, standardize, smoothing_fwhm, img.path @@ -146,9 +150,11 @@ def run_postprocessing_dataset( f"{attribute_name}_connectome", data=correlation_matrix ) + gc_log.info(f"Saved to:\n{output_path}") + if analysis_level == "group": - gc_log.info("create group connectome") + gc_log.info("Create group connectome") for desc in connectomes: average_connectome = np.mean( diff --git a/giga_connectome/workflow.py b/giga_connectome/workflow.py index 1d19e26..3b6e2e4 100644 --- a/giga_connectome/workflow.py +++ b/giga_connectome/workflow.py @@ -62,7 +62,7 @@ def workflow(args): "MNI152NLin6Asym" if is_ica_aroma(strategy) else "MNI152NLin2009cAsym" ) - gc_log.info("Indexing BIDS directory") + gc_log.info(f"Indexing BIDS directory:\n\t{bids_dir}") # create subject ts and connectomes # refactor the two cases into one @@ -81,7 +81,7 @@ def workflow(args): ) connectome_path = utils.check_path(connectome_path) - gc_log.info("Generate subject level connectomes") + gc_log.info(f"Generate subject level connectomes: sub-{subject}") run_postprocessing_dataset( strategy, diff --git a/pyproject.toml b/pyproject.toml index 66c5fa3..9577fc5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ dependencies = [ "templateflow < 23.0.0", "tqdm", "setuptools", + "rich", ] dynamic = ["version"] From a590c31114b8e1769405ab84ffd49ae81e9b1098 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Thu, 14 Dec 2023 16:41:35 +0100 Subject: [PATCH 5/9] minor fixes (#71) --- docs/source/conf.py | 2 +- docs/source/contributing.md | 4 +-- docs/source/usage.md | 50 ++++++++----------------------------- giga_connectome/run.py | 9 +++++-- pyproject.toml | 3 ++- 5 files changed, 22 insertions(+), 46 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index c5d70af..4d28c2d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -25,6 +25,7 @@ "sphinx.ext.autodoc", "sphinx.ext.autosummary", "sphinx.ext.napoleon", + "sphinxarg.ext", ] templates_path = ["_templates"] @@ -35,7 +36,6 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output html_theme = "sphinx_rtd_theme" -html_static_path = ["_static"] # -- Options for myst_parser ------------------------------------------------- myst_enable_extensions = ["colon_fence"] diff --git a/docs/source/contributing.md b/docs/source/contributing.md index 29aa39a..fd4fca8 100644 --- a/docs/source/contributing.md +++ b/docs/source/contributing.md @@ -75,7 +75,7 @@ The workflow is the same as code contributions, with some minor differences. 1. Install the `[doc]` dependencies. ```bash -pip install -e .[doc] +pip install -e '.[doc]' ``` 2. After making changes, build the docs locally: @@ -104,7 +104,7 @@ This tells the development team that your pull request is a "work-in-progress", One your PR is ready a member of the development team will review your changes to confirm that they can be merged into the main codebase. -## Making an release +## Making a release Currently this project is not pushed to PyPi. We simply tag the version on the repository so users can reference the version of installation. diff --git a/docs/source/usage.md b/docs/source/usage.md index 59edbc3..95271ed 100644 --- a/docs/source/usage.md +++ b/docs/source/usage.md @@ -1,48 +1,17 @@ # Usage Notes -```bash -usage: giga_connectome [-h] [-v] [--participant_label PARTICIPANT_LABEL [PARTICIPANT_LABEL ...]] [-w WORK_DIR] [--atlas ATLAS] - [--denoise-strategy DENOISE_STRATEGY] [--standardize {zscore,psc}] [--smoothing_fwhm SMOOTHING_FWHM] [--reindex-bids] - [--bids-filter-file BIDS_FILTER_FILE] - bids_dir output_dir {participant,group} - -Generate connectome based on denoising strategy for fmriprep processed dataset. - -positional arguments: - bids_dir The directory with the input dataset (e.g. fMRIPrep derivative)formatted according to the BIDS standard. - output_dir The directory where the output files should be stored. - {participant,group} Level of the analysis that will be performed. Only group level is allowed as we need to generate a dataset inclusive brain mask. - -optional arguments: - -h, --help show this help message and exit - -v, --version show program's version number and exit - --participant_label PARTICIPANT_LABEL [PARTICIPANT_LABEL ...] - The label(s) of the participant(s) that should be analyzed. The label corresponds to sub- from the BIDS spec (so it does not include 'sub-'). If this parameter is not provided all subjects should be analyzed. Multiple participants can be specified with a space separated list. - -w WORK_DIR, --work-dir WORK_DIR - Path where intermediate results should be stored. - --atlas ATLAS The choice of atlas for time series extraction. Default atlas choices are: 'Schaefer20187Networks, 'MIST', 'DiFuMo'. User can pass a path to a json file containing configuration for their own choice of atlas. The default is 'MIST'. - --denoise-strategy DENOISE_STRATEGY - The choice of post-processing for denoising. The default choices are: 'simple', 'simple+gsr', 'scrubbing.2', 'scrubbing.2+gsr', 'scrubbing.5', 'scrubbing.5+gsr', 'acompcor50', 'icaaroma'. User can pass a path to a json file containing configuration for their own choice of denoising strategy. The defaultis 'simple'. - --standardize {zscore,psc} - The choice of signal standardization. The choices are z score or percent signal change (psc). The default is 'zscore'. - --smoothing_fwhm SMOOTHING_FWHM - Size of the full-width at half maximum in millimeters of the spatial smoothing to apply to the signal. The default is 5.0. - --reindex-bids Reindex BIDS data set, even if layout has already been created. - --bids-filter-file BIDS_FILTER_FILE - A JSON file describing custom BIDS input filters using PyBIDS.We use the same format as described in fMRIPrep documentation: https://fmriprep.org/en/latest/faq.html#how-do-i-select-only-certain-files-to-be-input-to-fmriprepHowever, the query filed should always be 'bold' +## Command line interface +```{eval-rst} +.. argparse:: + :prog: giga_connectome + :module: giga_connectome.run + :func: global_parser ``` -When performing `participant` level analysis, the output is a HDF5 file per participant that was passed to `--participant_label` or all subjects under `bids_dir`. -The output file name is: `sub-_atlas-_desc-.h5` - -When performing `group` level analysis, the output is a HDF5 file per participant that was passed to `--participant_label` or all subjects under `bids_dir`. -The output file name is: `atlas-_desc-.h5` -The file will contain time series and connectomes of each subject, as well as group average connectomes. - ## Writing configuration files -All preset can be found in `giga_connectome/data` +All preset can be found in [`giga_connectome/data`](https://github.com/SIMEXP/giga_connectome/tree/main/giga_connectome/data). ### Denoising strategy @@ -62,13 +31,14 @@ In a `json` file, define the customised strategy in the following format: } ``` -See examples in `giga_connectome/data/denoise_strategy`. +See examples in [`giga_connectome/data/denoise_strategy`](https://github.com/SIMEXP/giga_connectome/tree/main/giga_connectome/data/denoise_strategy). ### Atlas After the atlas files are organised according to the [TemplateFlow](https://www.templateflow.org/python-client/0.7.1/naming.html) convention. A minimal set up should look like this: + ``` my_atlas/ └──tpl-MNI152NLin2009cAsym/ # template directory of a valid template name @@ -96,4 +66,4 @@ In a `json` file, define the customised atlas. We will use the atlas above as an } ``` -See examples in `giga_connectome/data/atlas`. +See examples in [`giga_connectome/data`](https://github.com/SIMEXP/giga_connectome/tree/main/giga_connectome/data). diff --git a/giga_connectome/run.py b/giga_connectome/run.py index b39365d..b265a32 100644 --- a/giga_connectome/run.py +++ b/giga_connectome/run.py @@ -4,8 +4,7 @@ from giga_connectome import __version__ -def main(argv=None): - """Entry point.""" +def global_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( formatter_class=argparse.RawTextHelpFormatter, description=( @@ -105,6 +104,12 @@ def main(argv=None): "pipeline (option A). The default is False.", action="store_true", ) + return parser + + +def main(argv=None): + """Entry point.""" + parser = global_parser() args = parser.parse_args(argv) diff --git a/pyproject.toml b/pyproject.toml index 66c5fa3..63ae89d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,8 @@ test = [ docs = [ "sphinx", "sphinx_rtd_theme", - "myst-parser" + "myst-parser", + "sphinx-argparse" ] # Aliases tests = ["giga_connectome[test]"] From 33d7f8545faf17723577bffa5a6fa9851fbb1dbe Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Thu, 14 Dec 2023 16:44:36 +0100 Subject: [PATCH 6/9] [MAINT] run pre-commit in CI to check style (#72) * run pre-commit in CI to check style * fix spaces --- .github/workflows/run_precommit.yml | 14 ++++++++++++++ CITATION.cff | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/run_precommit.yml diff --git a/.github/workflows/run_precommit.yml b/.github/workflows/run_precommit.yml new file mode 100644 index 0000000..a100dcb --- /dev/null +++ b/.github/workflows/run_precommit.yml @@ -0,0 +1,14 @@ +name: pre-commit + +on: + pull_request: + push: + branches: [main] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - uses: pre-commit/action@v3.0.0 diff --git a/CITATION.cff b/CITATION.cff index 132117c..cf6dda5 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -22,8 +22,8 @@ authors: email: htwangtw@gmail.com - family-names: Dessain given-names: Quentin - - family-names: Natasha + - family-names: Natasha given-names: Clarke - + license: MIT From d707ac6de41bd6347d1d2b252237dabe951127a2 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Thu, 14 Dec 2023 16:46:31 +0100 Subject: [PATCH 7/9] use tox to install test data (#73) --- .github/workflows/test.yml | 17 ++++---------- docs/source/contributing.md | 10 ++++++++- pyproject.toml | 1 + tox.ini | 45 +++++++++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 tox.ini diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0be3914..735a460 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,6 +43,9 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + - uses: actions/setup-python@v5 + - name: install tox + run: pip install tox - uses: actions/cache@v3 id: cache env: @@ -54,19 +57,7 @@ jobs: - if: ${{ steps.cache.outputs.cache-hit != 'true' }} name: Download fmriprep derivative of ds000017 id: download - run: | - mkdir -p /home/runner/work/giga_connectome/giga_connectome/giga_connectome/data/test_data - cd /home/runner/work/giga_connectome/giga_connectome/giga_connectome/data/test_data - wget --retry-connrefused \ - --waitretry=5 \ - --read-timeout=20 \ - --timeout=15 \ - -t 0 \ - -q \ - -O ds000017.tar.gz \ - "https://zenodo.org/record/8091903/files/ds000017-fmriprep22.0.1-downsampled-nosurface.tar.gz?download=1" - tar -xzf ds000017.tar.gz - rm ds000017.tar.gz + run: tox -e test_data build: runs-on: ubuntu-latest diff --git a/docs/source/contributing.md b/docs/source/contributing.md index fd4fca8..8e1fbb4 100644 --- a/docs/source/contributing.md +++ b/docs/source/contributing.md @@ -27,6 +27,14 @@ pip install -e .[dev] pre-commit install ``` +5. Install the data required for testing from zenodo + +This can be done using tox by running: + +```bash +tox -e test_data +``` + ## Contributing to code This is a very generic workflow. @@ -51,7 +59,7 @@ git checkout -b your_branch 5. Run the tests locally; you can run spectfic tests to speed up the process: ```bash -pytest -v giga_connectome/tests/tests/test_connectome.py::test_calculate_intranetwork_correlation +pytest -v giga_connectome/tests/test_connectome.py::test_calculate_intranetwork_correlation ``` 6. push your changes to your online fork. If this is the first commit, you might want to set up the remote tracking: diff --git a/pyproject.toml b/pyproject.toml index 63ae89d..74833e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dev = [ "black", "pre-commit", "giga_connectome[test]", + 'tox', ] test = [ "pytest", diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..280d5d8 --- /dev/null +++ b/tox.ini @@ -0,0 +1,45 @@ +; See https://tox.wiki/en +[tox] +requires = + tox>=4 +; run lint by default when just calling "tox" +env_list = lint + +; ENVIRONMENTS +; ------------ +[style] +description = common environment for style checkers (rely on pre-commit hooks) +skip_install = true +deps = + pre-commit + +; COMMANDS +; -------- +[testenv:lint] +description = run all linters and formatters +skip_install = true +deps = + {[style]deps} +commands = + pre-commit run --all-files --show-diff-on-failure {posargs:} + +[testenv:test_data] +description = install test data +skip_install = true +allowlist_externals = + mkdir + wget + tar + rm +commands = + mkdir -p giga_connectome/data/test_data + wget --retry-connrefused \ + --waitretry=5 \ + --read-timeout=20 \ + --timeout=15 \ + -t 0 \ + -q \ + -O giga_connectome/data/test_data/ds000017.tar.gz \ + "https://zenodo.org/record/8091903/files/ds000017-fmriprep22.0.1-downsampled-nosurface.tar.gz?download=1" + tar -xzf giga_connectome/data/test_data/ds000017.tar.gz -C giga_connectome/data/test_data/ + rm giga_connectome/data/test_data/ds000017.tar.gz From cfe904ec3cb7a77af48c5cc78072f405cbb27a91 Mon Sep 17 00:00:00 2001 From: Hao-Ting Wang Date: Thu, 14 Dec 2023 14:19:30 -0500 Subject: [PATCH 8/9] [FIX] accidentally deleted `main` --- giga_connectome/run.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/giga_connectome/run.py b/giga_connectome/run.py index 84079d9..2b1617d 100644 --- a/giga_connectome/run.py +++ b/giga_connectome/run.py @@ -115,6 +115,12 @@ def global_parser() -> argparse.ArgumentParser: type=int, nargs=1, ) + return parser + + +def main(argv=None): + """Entry point.""" + parser = global_parser() args = parser.parse_args(argv) From d62c9fdb43946de7668eb6156c666467400512f9 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Thu, 14 Dec 2023 20:38:49 +0100 Subject: [PATCH 9/9] bump nilearn version (#67) --- pyproject.toml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 74833e3..8a0efd3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ classifiers = [ ] dependencies = [ "h5py", - "nilearn", + "nilearn >=0.10.2", "pybids >=0.15.0, <0.16.0", "templateflow < 23.0.0", "tqdm", diff --git a/requirements.txt b/requirements.txt index 966be51..00e9d5b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ interface-meta==1.3.0 joblib==1.3.2 lxml==4.9.3 nibabel==5.2.0 -nilearn==0.9.2 +nilearn==0.10.2 num2words==0.5.13 numpy==1.26.2 packaging==23.2