[ENH] Add option for a BIDS filter file (#256)

Dockerfile

@@ -356,4 +356,5 @@ RUN  mkdir -p /sngl/data \
   && mkdir /sngl/scratch \
   && mkdir /sngl/spec \
   && mkdir /sngl/eddy \
+  && mkdir /sngl/filter \
   && chmod a+rwx /sngl/*

qsiprep/cli/run.py

@@ -39,14 +39,48 @@ def check_deps(workflow):
                       and which(node.interface._cmd.split()[0]) is None))
 
 
+def _filter_pybids_none_any(dct):
+    import bids
+    return {
+        k: bids.layout.Query.NONE
+        if v is None
+        else (bids.layout.Query.ANY if v == "*" else v)
+        for k, v in dct.items()
+    }
+
+
+def _bids_filter(value):
+    from json import loads
+    from bids.layout import Query
+
+    if value and Path(value).exists():
+        try:
+            filters = loads(Path(value).read_text(), object_hook=_filter_pybids_none_any)
+        except Exception as e:
+            raise Exception("Unable to parse BIDS filter file. Check that it is "
+                            "valid JSON.")
+    else:
+        raise Exception("Unable to load BIDS filter file " + value)
+
+    # unserialize pybids Query enum values
+    for acq, _filters in filters.items():
+        filters[acq] = {
+            k: getattr(Query, v[7:-4])
+            if not isinstance(v, Query) and "Query" in v
+            else v
+            for k, v in _filters.items()
+        }
+    return filters
+
+
 def get_parser():
     """Build parser object"""
     from ..__about__ import __version__
 
     verstr = 'qsiprep v{}'.format(__version__)
 
     parser = ArgumentParser(
-        description='qsiprep: q-Space Image PREProcessing workflows',
+        description='qsiprep: q-Space Image Preprocessing workflows',
         formatter_class=ArgumentDefaultsHelpFormatter)
 
     # Arguments as specified by BIDS-Apps
@@ -88,6 +122,16 @@ def get_parser():
                         help="path to a saved BIDS database directory",
                         type=Path,
                         action='store')
+    g_bids.add_argument(
+        "--bids-filter-file",
+        dest="bids_filters",
+        action="store",
+        type=_bids_filter,
+        metavar="FILE",
+        help="a JSON file describing custom BIDS input filters using PyBIDS. "
+        "For further details, please check out "
+        "https://fmriprep.readthedocs.io/en/latest/faq.html#"
+        "how-do-I-select-only-certain-files-to-be-input-to-fMRIPrep")
 
     # arguments for reconstructing QSI data
     g_ireports = parser.add_argument_group('Options for interactive report outputs')
@@ -767,6 +811,7 @@ def build_qsiprep_workflow(opts, retval):
                     "sourcedata",
                     "models",
                     re.compile(r"^\.")))
+
     subject_list = collect_participants(
         layout, participant_label=opts.participant_label)
     retval['subject_list'] = subject_list
@@ -886,6 +931,7 @@ def build_qsiprep_workflow(opts, retval):
         ignore=opts.ignore,
         hires=False,
         freesurfer=opts.do_reconall,
+        bids_filters=opts.bids_filters,
         debug=opts.sloppy,
         low_mem=opts.low_mem,
         dwi_only=opts.dwi_only,

qsiprep/niworkflows/utils/bids.py

@@ -12,7 +12,6 @@
 import re
 import simplejson as json
 
-from bids import BIDSLayout
 from .misc import splitext
 
 __all__ = ['BIDS_NAME']
@@ -89,44 +88,6 @@ def collect_participants(bids_dir, participant_label=None, strict=False):
     return found_label
 
 
-def collect_data(dataset, participant_label, task=None, echo=None):
-    """
-    Uses pybids to retrieve the input data for a given participant
-    """
-    layout = BIDSLayout(dataset, exclude=['derivatives', 'sourcedata'])
-    queries = {
-        'fmap': {'subject': participant_label, 'modality': 'fmap',
-                 'extensions': ['nii', 'nii.gz']},
-        'bold': {'subject': participant_label, 'modality': 'func', 'type': 'bold',
-                 'extensions': ['nii', 'nii.gz']},
-        'sbref': {'subject': participant_label, 'modality': 'func', 'type': 'sbref',
-                  'extensions': ['nii', 'nii.gz']},
-        'flair': {'subject': participant_label, 'modality': 'anat', 'type': 'FLAIR',
-                  'extensions': ['nii', 'nii.gz']},
-        't2w': {'subject': participant_label, 'modality': 'anat', 'type': 'T2w',
-                'extensions': ['nii', 'nii.gz']},
-        't1w': {'subject': participant_label, 'modality': 'anat', 'type': 'T1w',
-                'extensions': ['nii', 'nii.gz']},
-        'roi': {'subject': participant_label, 'modality': 'anat', 'type': 'roi',
-                'extensions': ['nii', 'nii.gz']},
-    }
-
-    if task:
-        queries['bold']['task'] = task
-
-    if echo:
-        queries['bold']['echo'] = echo
-
-    subj_data = {modality: [x.filename for x in layout.get(**query)]
-                 for modality, query in queries.items()}
-
-    # Special case: multi-echo BOLD, grouping echos
-    if any(['_echo-' in bold for bold in subj_data['bold']]):
-        subj_data['bold'] = group_multiecho(subj_data['bold'])
-
-    return subj_data, layout
-
-
 def get_metadata_for_nifti(in_file):
     """Fetch metadata for a given nifti file
 

qsiprep/utils/bids.py

@@ -77,7 +77,7 @@ def collect_participants(bids_dir, participant_label=None, strict=False,
     if isinstance(bids_dir, BIDSLayout):
         layout = bids_dir
     else:
-        layout = BIDSLayout(str(bids_dir), validate=bids_validate)
+        raise Exception("A layout is required")
 
     all_participants = set(layout.get_subjects())
 
@@ -120,7 +120,7 @@ def collect_participants(bids_dir, participant_label=None, strict=False,
     return found_label
 
 
-def collect_data(bids_dir, participant_label, task=None, bids_validate=True):
+def collect_data(bids_dir, participant_label, filters=None, bids_validate=True):
     """
     Uses pybids to retrieve the input data for a given participant
 
@@ -139,11 +139,21 @@ def collect_data(bids_dir, participant_label, task=None, bids_validate=True):
         'roi': {'datatype': 'anat', 'suffix': 'roi'},
         'dwi': {'datatype': 'dwi', 'suffix': 'dwi'}
     }
+    bids_filters = filters or {}
+    for acq, entities in bids_filters.items():
+        queries[acq].update(entities)
 
     subj_data = {
-        dtype: sorted(layout.get(return_type='file', subject=participant_label,
-                                 extension=['nii', 'nii.gz'], **query))
-        for dtype, query in queries.items()}
+        dtype: sorted(
+            layout.get(
+                return_type="file",
+                subject=participant_label,
+                extension=["nii", "nii.gz"],
+                **query,
+            )
+        )
+        for dtype, query in queries.items()
+    }
 
     return subj_data, layout
 

qsiprep/workflows/base.py

@@ -45,7 +45,7 @@ def init_qsiprep_wf(
         subject_list, run_uuid, work_dir, output_dir, bids_dir, ignore, debug, low_mem, anat_only,
         dwi_only, longitudinal, b0_threshold, hires, denoise_before_combining, dwi_denoise_window,
         denoise_method, unringing_method, dwi_no_biascorr, no_b0_harmonization, output_resolution,
-        infant_mode, combine_all_dwis, distortion_group_merge, omp_nthreads,
+        infant_mode, combine_all_dwis, distortion_group_merge, omp_nthreads, bids_filters,
         force_spatial_normalization, skull_strip_template, skull_strip_fixed_seed, freesurfer,
         hmc_model, impute_slice_threshold, hmc_transform, shoreline_iters, eddy_config,
         write_local_bvecs, output_spaces, template, motion_corr_to, b0_to_t1w_transform,
@@ -68,6 +68,7 @@ def init_qsiprep_wf(
                               output_dir='.',
                               bids_dir='.',
                               ignore=[],
+                              bids_filters=None,
                               debug=False,
                               low_mem=False,
                               dwi_only=False,
@@ -231,6 +232,7 @@ def init_qsiprep_wf(
             reportlets_dir=reportlets_dir,
             output_dir=output_dir,
             bids_dir=bids_dir,
+            bids_filters=bids_filters,
             ignore=ignore,
             debug=debug,
             low_mem=low_mem,
@@ -286,15 +288,18 @@ def init_qsiprep_wf(
 
 
 def init_single_subject_wf(
-        subject_id, name, reportlets_dir, output_dir, bids_dir, ignore, debug, write_local_bvecs,
-        low_mem, dwi_only, anat_only, longitudinal, b0_threshold, denoise_before_combining,
-        dwi_denoise_window, denoise_method, unringing_method, dwi_no_biascorr, no_b0_harmonization,
-        infant_mode, combine_all_dwis, distortion_group_merge, omp_nthreads, skull_strip_template,
-        force_spatial_normalization, skull_strip_fixed_seed, freesurfer, hires, output_spaces,
-        template, output_resolution, prefer_dedicated_fmaps, motion_corr_to, b0_to_t1w_transform,
-        intramodal_template_iters, intramodal_template_transform, hmc_model, hmc_transform,
-        shoreline_iters, eddy_config, impute_slice_threshold, fmap_bspline, fmap_demean, use_syn,
-        force_syn):
+        subject_id, name, reportlets_dir, output_dir, bids_dir, ignore, debug,
+        write_local_bvecs, low_mem, dwi_only, anat_only, longitudinal,
+        b0_threshold, denoise_before_combining, bids_filters,
+        dwi_denoise_window, denoise_method, unringing_method, dwi_no_biascorr,
+        no_b0_harmonization, infant_mode, combine_all_dwis,
+        distortion_group_merge, omp_nthreads, skull_strip_template,
+        force_spatial_normalization, skull_strip_fixed_seed, freesurfer, hires,
+        output_spaces, template, output_resolution, prefer_dedicated_fmaps,
+        motion_corr_to, b0_to_t1w_transform, intramodal_template_iters,
+        intramodal_template_transform, hmc_model, hmc_transform,
+        shoreline_iters, eddy_config, impute_slice_threshold, fmap_bspline,
+        fmap_demean, use_syn, force_syn):
     """
     This workflow organizes the preprocessing pipeline for a single subject.
     It collects and reports information about the subject, and prepares
@@ -317,6 +322,7 @@ def init_single_subject_wf(
             reportlets_dir='.',
             output_dir='.',
             bids_dir='.',
+            bids_filters=None,
             ignore=[],
             debug=False,
             low_mem=False,
@@ -425,7 +431,7 @@ def init_single_subject_wf(
 
         template : str
             Name of template targeted by ``template`` output space
-        hmc_model : 'none', '3dSHORE' or 'MAPMRI'
+        hmc_model : 'none', '3dSHORE' or 'eddy'
             Model used to generate target images for head motion correction. If 'none'
             the transform from the nearest b0 will be used.
         hmc_transform : "Rigid" or "Affine"
@@ -475,7 +481,7 @@ def init_single_subject_wf(
         layout = None
         LOGGER.warning("Building a test workflow")
     else:
-        subject_data, layout = collect_data(bids_dir, subject_id)
+        subject_data, layout = collect_data(bids_dir, subject_id, filters=bids_filters)
 
     if anat_only and dwi_only:
         raise Exception("--anat-only and --dwi-only are mutually exclusive.")

wrapper/qsiprep_docker.py

@@ -238,6 +238,12 @@ def get_parser():
                         action='store',
                         type=os.path.abspath)
 
+    # For BIDS filters
+    parser.add_argument('--bids-filter-file', '--bids_filter_file',
+                        required=False,
+                        action='store',
+                        type=os.path.abspath)
+
     parser.add_argument('-h', '--help', action='store_true',
                         help="show this help message and exit")
     parser.add_argument('--version', action='store_true',
@@ -388,6 +394,11 @@ def main():
             main_args.extend(['--recon-spec', '/sngl/spec/spec.json'])
         else:
             main_args.extend(['--recon-spec', opts.recon_spec])
+    if opts.bids_filter_file:
+        if os.path.exists(opts.bids_filter_file):
+            command.extend(['-v', ':'.join((opts.bids_filter_file,
+                            '/sngl/filter/filter.json', 'ro'))])
+            main_args.extend(['--bids-filter-file', '/sngl/filter/filter.json'])
     if opts.eddy_config:
         command.extend(['-v', ':'.join((opts.eddy_config, '/sngl/eddy/eddy_config.json', 'ro'))])
         main_args.extend(['--eddy-config', '/sngl/eddy/eddy_config.json'])

wrapper/qsiprep_singularity.py

@@ -213,6 +213,10 @@ def get_parser():
                         required=False,
                         action='store',
                         type=os.path.abspath)
+    parser.add_argument('--bids-filter-file', '--bids_filter_file',
+                        required=False,
+                        action='store',
+                        type=os.path.abspath)
 
     parser.add_argument('-h', '--help', action='store_true',
                         help="show this help message and exit")
@@ -379,6 +383,11 @@ def main():
         mounted_config = "/sngl/eddy/" + config_fname
         command.extend(['-B', ':'.join((config_dir, '/sngl/eddy'))])
         main_args.extend(['--eddy-config', mounted_config])
+    if opts.bids_filter_file:
+        filter_dir, filter_fname = op.split(opts.bids_filter_file)
+        mounted_filter = "/sngl/filter/" + filter_fname
+        command.extend(['-B', ':'.join((filter_dir, '/sngl/filter'))])
+        main_args.extend(['--bids-filter-file', mounted_filter])
     if opts.output_dir:
         mkdir(opts.output_dir)
         command.extend(['-B', ':'.join((opts.output_dir, '/sngl/out'))])