[ENH] Allow the T1-Linear pipeline to go over (subject, session) couples that do not have data

clinica/pipelines/t1_linear/anat_linear_pipeline.py

@@ -75,7 +75,10 @@ def get_processed_images(
         image_ids: List[str] = []
         if caps_directory.is_dir():
             cropped_files, _ = clinica_file_reader(
-                subjects, sessions, caps_directory, T1W_LINEAR_CROPPED, False
+                subjects,
+                sessions,
+                caps_directory,
+                T1W_LINEAR_CROPPED,
             )
             image_ids = extract_image_ids(cropped_files)
         return image_ids
@@ -117,7 +120,7 @@ def _build_input_node(self):
         from clinica.utils.filemanip import extract_subjects_sessions_from_filename
         from clinica.utils.image import get_mni_template
         from clinica.utils.input_files import T1W_NII, Flair_T2W_NII
-        from clinica.utils.inputs import clinica_file_reader
+        from clinica.utils.inputs import clinica_file_filter
         from clinica.utils.stream import cprint
         from clinica.utils.ux import print_images_to_process
 
@@ -149,17 +152,12 @@ def _build_input_node(self):
         # Inputs from anat/ folder
         # ========================
         # anat image file:
-        try:
-            file = T1W_NII if self.name == "t1-linear" else Flair_T2W_NII
-            anat_files, _ = clinica_file_reader(
-                self.subjects, self.sessions, self.bids_directory, file
-            )
-        except ClinicaException as e:
-            err = (
-                "Clinica faced error(s) while trying to read files in your BIDS directory.\n"
-                + str(e)
-            )
-            raise ClinicaBIDSError(err)
+        file = T1W_NII if self.name == "t1-linear" else Flair_T2W_NII
+
+        anat_files = clinica_file_filter(
+            self.subjects, self.sessions, self.bids_directory, file
+        )
+        # todo : what happens if empty self.subjects ?
 
         if len(self.subjects):
             print_images_to_process(self.subjects, self.sessions)

clinica/utils/inputs.py

@@ -281,7 +281,7 @@ def find_sub_ses_pattern_path(
 ) -> None:
     """Appends the output path corresponding to subject, session and pattern in results.
 
-    If an error is encountered, its corresponding message is added to the list `error_encountered`.
+    If an error is encountered, its (subject,session) couple is added to the list `errors_encountered`.
 
     Parameters
     ----------
@@ -345,12 +345,10 @@ def find_sub_ses_pattern_path(
             )
             results.append(selected)
         else:
-            error_str = f"\t*  ({subject} | {session}): More than 1 file found:\n"
-            for found_file in current_glob_found:
-                error_str += f"\t\t{found_file}\n"
-            error_encountered.append(error_str)
+            # todo : RQ : now we do not know why the invalid files are considered invalid
+            error_encountered += [(subject, session)]
     elif len(current_glob_found) == 0:
-        error_encountered.append(f"\t* ({subject} | {session}): No file found\n")
+        error_encountered += [(subject, session)]
     # Otherwise the file found is added to the result
     else:
         results.append(current_glob_found[0])
@@ -569,29 +567,65 @@ def _check_information(information: Dict) -> None:
             )
 
 
-def _format_errors(errors: List, information: Dict) -> str:
-    error_message = (
+def clinica_file_filter(
+    subjects: List[str],
+    sessions: List[str],
+    input_directory: Path,
+    information: Dict,
+    n_procs: Optional[int] = 1,
+) -> List[str]:
+    from clinica.utils.stream import cprint
+
+    # todo : test ?
+    # todo : typing ?
+    files, errors = clinica_file_reader(
+        subjects, sessions, input_directory, information, n_procs
+    )
+    cprint(_format_errors(errors, information), "warning")
+    _remove_sub_ses_from_list(subjects, sessions, errors)
+    return files
+
+
+def _format_errors(errors: List[Tuple[str, str]], information: Dict) -> str:
+    message = (
         f"Clinica encountered {len(errors)} "
         f"problem(s) while getting {information['description']}:\n"
     )
     if "needed_pipeline" in information and information["needed_pipeline"]:
-        error_message += (
+        message += (
             "Please note that the following clinica pipeline(s) must "
             f"have run to obtain these files: {information['needed_pipeline']}\n"
         )
-    error_message += "\n".join(errors)
+    if errors:
+        message += "".join(f"\t* ({e[0]} | {e[1]})\n" for e in errors)
+        message += "Clinica could not identify which file to use (missing or too many) for these sessions. They will not be processed."
+    return message
+
 
-    return error_message
+def _remove_sub_ses_from_list(
+    list_subjects: List[str],
+    list_sessions: List[str],
+    wrong_sub_ses: List[Tuple[str, str]],
+) -> None:
+    for sub, ses in wrong_sub_ses:
+        sub_indexes = [i for i, subject in enumerate(list_subjects) if subject == sub]
+        session_indexes = [
+            i for i, session in enumerate(list_sessions) if session == ses
+        ]
+        to_remove = list(set(sub_indexes) & set(session_indexes))
+        to_remove.sort(reverse=True)
+        for index in to_remove:
+            list_subjects.pop(index)
+            list_sessions.pop(index)
 
 
 def clinica_file_reader(
     subjects: List[str],
     sessions: List[str],
     input_directory: os.PathLike,
     information: Dict,
-    raise_exception: Optional[bool] = True,
     n_procs: Optional[int] = 1,
-):
+) -> Tuple[List[str], List[Tuple[str, str]]]:
     """Read files in BIDS or CAPS directory based on participant ID(s).
 
     This function grabs files relative to a subject and session list according to a glob pattern (using *)
@@ -616,10 +650,6 @@ def clinica_file_reader(
             - `needed_pipeline` : Optional. String describing the pipeline(s)
               needed to obtain the related file.
 
-    raise_exception : bool, optional
-        If True, an exception is raised if errors happen. If not, we return the file
-        list as it is. Default=True.
-
     n_procs : int, optional
         Number of cores used to fetch files in parallel.
         If set to 1, subjects and sessions will be processed sequentially.
@@ -630,9 +660,6 @@ def clinica_file_reader(
     results : List[str]
         List of files respecting the subject/session order provided in input.
 
-    error_message : str
-        Error message which contains all errors encountered while reading the files.
-
     Raises
     ------
     TypeError
@@ -725,12 +752,12 @@ def clinica_file_reader(
     or even more precise: 't1/freesurfer_cross_sectional/sub-*_ses-*/surf/rh.white'
     It then gives: ['/caps/subjects/sub-ADNI011S4105/ses-M00/t1/freesurfer_cross_sectional/sub-ADNI011S4105_ses-M00/surf/rh.white']
     """
-    from clinica.utils.exceptions import ClinicaBIDSError, ClinicaCAPSError
+    # todo : change function description
 
+    input_directory = Path(input_directory)
     _check_information(information)
     pattern = information["pattern"]
 
-    input_directory = Path(input_directory)
     is_bids = determine_caps_or_bids(input_directory)
     if is_bids:
         check_bids_folder(input_directory)
@@ -741,26 +768,17 @@ def clinica_file_reader(
         raise ValueError("Subjects and sessions must have the same length.")
 
     if len(subjects) == 0:
-        return [], ""
+        return [], []
 
     file_reader = _read_files_parallel if n_procs > 1 else _read_files_sequential
-    results, errors_encountered = file_reader(
+    return file_reader(
         input_directory,
         subjects,
         sessions,
         is_bids,
         pattern,
         n_procs=n_procs,
     )
-    error_message = _format_errors(errors_encountered, information)
-
-    if len(errors_encountered) > 0 and raise_exception:
-        if is_bids:
-            raise ClinicaBIDSError(error_message)
-        else:
-            raise ClinicaCAPSError(error_message)
-
-    return results, error_message
 
 
 def _read_files_parallel(
@@ -770,7 +788,7 @@ def _read_files_parallel(
     is_bids: bool,
     pattern: str,
     n_procs: int,
-) -> Tuple[List[str], List[str]]:
+) -> Tuple[List[str], List[Tuple[str, str]]]:
     from multiprocessing import Manager
 
     from joblib import Parallel, delayed
@@ -791,7 +809,7 @@ def _read_files_parallel(
         for sub, ses in zip(subjects, sessions)
     )
     results = list(shared_results)
-    errors_encountered = list(shared_errors_encountered)
+    errors_encountered = list(shared_errors_encountered)  # todo : needed ?
     return results, errors_encountered
 
 
@@ -802,7 +820,7 @@ def _read_files_sequential(
     is_bids: bool,
     pattern: str,
     **kwargs,
-) -> Tuple[List[str], List[str]]:
+) -> Tuple[List[str], List[Tuple[str, str]]]:
     errors_encountered, results = [], []
     for sub, ses in zip(subjects, sessions):
         find_sub_ses_pattern_path(
@@ -847,29 +865,26 @@ def clinica_list_of_files_reader(
     list_found_files : List[List[str]]
         List of lists of found files following order of `list_information`
     """
-    from .exceptions import ClinicaBIDSError, ClinicaException
+    from .exceptions import ClinicaBIDSError
 
     all_errors = []
     list_found_files = []
     for info_file in list_information:
-        try:
-            list_found_files.append(
-                clinica_file_reader(
-                    participant_ids,
-                    session_ids,
-                    bids_or_caps_directory,
-                    info_file,
-                    True,
-                )[0]
-            )
-        except ClinicaException as e:
-            list_found_files.append([])
-            all_errors.append(e)
+        files, errors = clinica_file_reader(
+            participant_ids,
+            session_ids,
+            bids_or_caps_directory,
+            info_file,
+        )
+        # todo : should it work like that ?? maybe after change need to filter subject list ?
+
+        list_found_files.append(files)
+        all_errors += errors
 
     if len(all_errors) > 0 and raise_exception:
         error_message = "Clinica faced error(s) while trying to read files in your BIDS or CAPS directory.\n"
-        for msg in all_errors:
-            error_message += str(msg)
+        for error in all_errors:
+            error_message += f"\n\t({error[0]} | {error[1]})"
         raise ClinicaBIDSError(error_message)
 
     return list_found_files