diff --git a/examples/02_meta-analyses/02_plot_ibma.py b/examples/02_meta-analyses/02_plot_ibma.py index 01bbfcbb3..1eac8a7c0 100644 --- a/examples/02_meta-analyses/02_plot_ibma.py +++ b/examples/02_meta-analyses/02_plot_ibma.py @@ -86,7 +86,7 @@ # ----------------------------------------------------------------------------- from nimare.meta.ibma import Fishers -meta = Fishers(resample=True) +meta = Fishers() results = meta.fit(dset) plot_stat_map( diff --git a/examples/02_meta-analyses/06_plot_compare_ibma_and_cbma.py b/examples/02_meta-analyses/06_plot_compare_ibma_and_cbma.py index 3e184cace..433389b51 100644 --- a/examples/02_meta-analyses/06_plot_compare_ibma_and_cbma.py +++ b/examples/02_meta-analyses/06_plot_compare_ibma_and_cbma.py @@ -58,8 +58,7 @@ ############################################################################### # DerSimonian-Laird (IBMA) # ----------------------------------------------------------------------------- -# We must resample the image data to the same MNI template as the Dataset. -meta_ibma = DerSimonianLaird(resample=True) +meta_ibma = DerSimonianLaird() ibma_results = meta_ibma.fit(dset) plot_stat_map( ibma_results.get_map("z"), diff --git a/nimare/meta/ibma.py b/nimare/meta/ibma.py index a9d6c1ab7..886f31b2f 100755 --- a/nimare/meta/ibma.py +++ b/nimare/meta/ibma.py @@ -15,6 +15,7 @@ from nimare import _version from nimare.estimator import Estimator +from nimare.meta.utils import _apply_liberal_mask from nimare.transforms import p_to_z, t_to_z from nimare.utils import _boolean_unmask, _check_ncores, get_masker @@ -44,12 +45,15 @@ class IBMAEstimator(Estimator): def __init__( self, + aggressive_mask=True, memory=Memory(location=None, verbose=0), memory_level=0, *, mask=None, **kwargs, ): + self.aggressive_mask = aggressive_mask + if mask is not None: mask = get_masker(mask, memory=memory, memory_level=memory_level) self.masker = mask @@ -79,16 +83,21 @@ def _preprocess_input(self, dataset): if isinstance(mask_img, str): mask_img = nib.load(mask_img) - # Ensure that protected values are not included among _required_inputs - assert "aggressive_mask" not in self._required_inputs.keys(), "This is a protected name." + if self.aggressive_mask: + # Ensure that protected values are not included among _required_inputs + assert ( + "aggressive_mask" not in self._required_inputs.keys() + ), "This is a protected name." - if "aggressive_mask" in self.inputs_.keys(): - LGR.warning("Removing existing 'aggressive_mask' from Estimator.") - self.inputs_.pop("aggressive_mask") + if "aggressive_mask" in self.inputs_.keys(): + LGR.warning("Removing existing 'aggressive_mask' from Estimator.") + self.inputs_.pop("aggressive_mask") + else: + # A dictionary to collect data, to be further reduced by the liberal mask. + self.inputs_["data_bags"] = {} # A dictionary to collect masked image data, to be further reduced by the aggressive mask. temp_image_inputs = {} - for name, (type_, _) in self._required_inputs.items(): if type_ == "image": # Resampling will only occur if shape/affines are different @@ -105,23 +114,29 @@ def _preprocess_input(self, dataset): # Mask required input images using either the dataset's mask or the estimator's. temp_arr = masker.transform(img4d) - # An intermediate step to mask out bad voxels. - # Can be dropped once PyMARE is able to handle masked arrays or missing data. - nonzero_voxels_bool = np.all(temp_arr != 0, axis=0) - nonnan_voxels_bool = np.all(~np.isnan(temp_arr), axis=0) - good_voxels_bool = np.logical_and(nonzero_voxels_bool, nonnan_voxels_bool) - data = masker.transform(img4d) - temp_image_inputs[name] = data - if "aggressive_mask" not in self.inputs_.keys(): - self.inputs_["aggressive_mask"] = good_voxels_bool + if self.aggressive_mask: + # An intermediate step to mask out bad voxels. + # Can be dropped once PyMARE is able to handle masked arrays or missing data. + nonzero_voxels_bool = np.all(temp_arr != 0, axis=0) + nonnan_voxels_bool = np.all(~np.isnan(temp_arr), axis=0) + good_voxels_bool = np.logical_and(nonzero_voxels_bool, nonnan_voxels_bool) + + if "aggressive_mask" not in self.inputs_.keys(): + self.inputs_["aggressive_mask"] = good_voxels_bool + else: + # Remove any voxels that are bad in any image-based inputs + self.inputs_["aggressive_mask"] = np.logical_or( + self.inputs_["aggressive_mask"], + good_voxels_bool, + ) else: - # Remove any voxels that are bad in any image-based inputs - self.inputs_["aggressive_mask"] = np.logical_or( - self.inputs_["aggressive_mask"], - good_voxels_bool, - ) + self.inputs_[name] = data # This data is saved only to use in Reports + data_bags = zip(*_apply_liberal_mask(data)) + + keys = ["values", "voxel_mask", "study_mask"] + self.inputs_["data_bags"][name] = [dict(zip(keys, bag)) for bag in data_bags] # Further reduce image-based inputs to remove "bad" voxels # (voxels with zeros or NaNs in any studies) @@ -146,6 +161,19 @@ class Fishers(IBMAEstimator): This method is described in :footcite:t:`fisher1946statistical`. + .. versionchanged:: 0.2.1 + + * New parameter: ``aggressive_mask``, to control whether to use an aggressive mask. + + Parameters + ---------- + aggressive_mask : :obj:`bool`, optional + Voxels with a value of zero of NaN in any of the input maps will be removed + from the analysis. + If False, all voxels are included by running a separate analysis on bags + of voxels that belong that have a valid value across the same studies. + Default is True. + Notes ----- Requires ``z`` images. @@ -155,6 +183,7 @@ class Fishers(IBMAEstimator): ============== =============================================================================== "z" Z-statistic map from one-sample test. "p" P-value map from one-sample test. + "dof" Degrees of freedom map from one-sample test. ============== =============================================================================== Warnings @@ -162,9 +191,11 @@ class Fishers(IBMAEstimator): Masking approaches which average across voxels (e.g., NiftiLabelsMaskers) will result in invalid results. It cannot be used with these types of maskers. - All image-based meta-analysis estimators adopt an aggressive masking + By default, all image-based meta-analysis estimators adopt an aggressive masking strategy, in which any voxels with a value of zero in any of the input maps - will be removed from the analysis. + will be removed from the analysis. Setting ``aggressive_mask=False`` will + instead run tha analysis in bags of voxels that have a valid value across + the same studies. References ---------- @@ -198,14 +229,35 @@ def _fit(self, dataset): "will produce invalid results." ) - pymare_dset = pymare.Dataset(y=self.inputs_["z_maps"]) - est = pymare.estimators.FisherCombinationTest() - est.fit_dataset(pymare_dset) - est_summary = est.summary() - maps = { - "z": _boolean_unmask(est_summary.z.squeeze(), self.inputs_["aggressive_mask"]), - "p": _boolean_unmask(est_summary.p.squeeze(), self.inputs_["aggressive_mask"]), - } + if self.aggressive_mask: + pymare_dset = pymare.Dataset(y=self.inputs_["z_maps"]) + est = pymare.estimators.FisherCombinationTest() + est.fit_dataset(pymare_dset) + est_summary = est.summary() + + z_map = _boolean_unmask(est_summary.z.squeeze(), self.inputs_["aggressive_mask"]) + p_map = _boolean_unmask(est_summary.p.squeeze(), self.inputs_["aggressive_mask"]) + dof_map = np.tile( + self.inputs_["z_maps"].shape[0] - 1, + self.inputs_["z_maps"].shape[1], + ).astype(np.int32) + dof_map = _boolean_unmask(dof_map, self.inputs_["aggressive_mask"]) + + else: + n_total_voxels = self.inputs_["z_maps"].shape[1] + z_map = np.zeros(n_total_voxels, dtype=float) + p_map = np.zeros(n_total_voxels, dtype=float) + dof_map = np.zeros(n_total_voxels, dtype=np.int32) + for bag in self.inputs_["data_bags"]["z_maps"]: + pymare_dset = pymare.Dataset(y=bag["values"]) + est = pymare.estimators.FisherCombinationTest() + est.fit_dataset(pymare_dset) + est_summary = est.summary() + z_map[bag["voxel_mask"]] = est_summary.z.squeeze() + p_map[bag["voxel_mask"]] = est_summary.p.squeeze() + dof_map[bag["voxel_mask"]] = bag["values"].shape[0] - 1 + + maps = {"z": z_map, "p": p_map, "dof": dof_map} description = self._generate_description() return maps, {}, description @@ -218,8 +270,18 @@ class Stouffers(IBMAEstimator): This method is described in :footcite:t:`stouffer1949american`. + .. versionchanged:: 0.2.1 + + * New parameter: ``aggressive_mask``, to control whether to use an aggressive mask. + Parameters ---------- + aggressive_mask : :obj:`bool`, optional + Voxels with a value of zero of NaN in any of the input maps will be removed + from the analysis. + If False, all voxels are included by running a separate analysis on bags + of voxels that belong that have a valid value across the same studies. + Default is True. use_sample_size : :obj:`bool`, optional Whether to use sample sizes for weights (i.e., "weighted Stouffer's") or not, as described in :footcite:t:`zaykin2011optimally`. @@ -234,6 +296,7 @@ class Stouffers(IBMAEstimator): ============== =============================================================================== "z" Z-statistic map from one-sample test. "p" P-value map from one-sample test. + "dof" Degrees of freedom map from one-sample test. ============== =============================================================================== Warnings @@ -241,9 +304,11 @@ class Stouffers(IBMAEstimator): Masking approaches which average across voxels (e.g., NiftiLabelsMaskers) will result in invalid results. It cannot be used with these types of maskers. - All image-based meta-analysis estimators adopt an aggressive masking + By default, all image-based meta-analysis estimators adopt an aggressive masking strategy, in which any voxels with a value of zero in any of the input maps - will be removed from the analysis. + will be removed from the analysis. Setting ``aggressive_mask=False`` will + instead run tha analysis in bags of voxels that have a valid value across + the same studies. References ---------- @@ -292,23 +357,52 @@ def _fit(self, dataset): "will produce invalid results." ) - est = pymare.estimators.StoufferCombinationTest() + if self.aggressive_mask: + est = pymare.estimators.StoufferCombinationTest() + + if self.use_sample_size: + sample_sizes = np.array([np.mean(n) for n in self.inputs_["sample_sizes"]]) + weights = np.sqrt(sample_sizes) + weight_maps = np.tile(weights, (self.inputs_["z_maps"].shape[1], 1)).T + pymare_dset = pymare.Dataset(y=self.inputs_["z_maps"], v=weight_maps) + else: + pymare_dset = pymare.Dataset(y=self.inputs_["z_maps"]) + + est.fit_dataset(pymare_dset) + est_summary = est.summary() + + z_map = _boolean_unmask(est_summary.z.squeeze(), self.inputs_["aggressive_mask"]) + p_map = _boolean_unmask(est_summary.p.squeeze(), self.inputs_["aggressive_mask"]) + dof_map = np.tile( + self.inputs_["z_maps"].shape[0] - 1, self.inputs_["z_maps"].shape[1] + ).astype(np.int32) + dof_map = _boolean_unmask(dof_map, self.inputs_["aggressive_mask"]) - if self.use_sample_size: - sample_sizes = np.array([np.mean(n) for n in self.inputs_["sample_sizes"]]) - weights = np.sqrt(sample_sizes) - weight_maps = np.tile(weights, (self.inputs_["z_maps"].shape[1], 1)).T - pymare_dset = pymare.Dataset(y=self.inputs_["z_maps"], v=weight_maps) else: - pymare_dset = pymare.Dataset(y=self.inputs_["z_maps"]) + n_total_voxels = self.inputs_["z_maps"].shape[1] + z_map = np.zeros(n_total_voxels, dtype=float) + p_map = np.zeros(n_total_voxels, dtype=float) + dof_map = np.zeros(n_total_voxels, dtype=np.int32) + for bag in self.inputs_["data_bags"]["z_maps"]: + est = pymare.estimators.StoufferCombinationTest() + + if self.use_sample_size: + study_mask = bag["study_mask"] + sample_sizes_ex = [self.inputs_["sample_sizes"][study] for study in study_mask] + sample_sizes = np.array([np.mean(n) for n in sample_sizes_ex]) + weights = np.sqrt(sample_sizes) + weight_maps = np.tile(weights, (bag["values"].shape[1], 1)).T + pymare_dset = pymare.Dataset(y=bag["values"], v=weight_maps) + else: + pymare_dset = pymare.Dataset(y=bag["values"]) - est.fit_dataset(pymare_dset) - est_summary = est.summary() + est.fit_dataset(pymare_dset) + est_summary = est.summary() + z_map[bag["voxel_mask"]] = est_summary.z.squeeze() + p_map[bag["voxel_mask"]] = est_summary.p.squeeze() + dof_map[bag["voxel_mask"]] = bag["values"].shape[0] - 1 - maps = { - "z": _boolean_unmask(est_summary.z.squeeze(), self.inputs_["aggressive_mask"]), - "p": _boolean_unmask(est_summary.p.squeeze(), self.inputs_["aggressive_mask"]), - } + maps = {"z": z_map, "p": p_map, "dof": dof_map} description = self._generate_description() return maps, {}, description @@ -317,6 +411,10 @@ def _fit(self, dataset): class WeightedLeastSquares(IBMAEstimator): """Weighted least-squares meta-regression. + .. versionchanged:: 0.2.1 + + * New parameter: ``aggressive_mask``, to control whether to use an aggressive mask. + .. versionchanged:: 0.0.12 * Add "se" to outputs. @@ -336,6 +434,12 @@ class WeightedLeastSquares(IBMAEstimator): Parameters ---------- + aggressive_mask : :obj:`bool`, optional + Voxels with a value of zero of NaN in any of the input maps will be removed + from the analysis. + If False, all voxels are included by running a separate analysis on bags + of voxels that belong that have a valid value across the same studies. + Default is True. tau2 : :obj:`float` or 1D :class:`numpy.ndarray`, optional Assumed/known value of tau^2. Must be >= 0. Default is 0. @@ -350,6 +454,7 @@ class WeightedLeastSquares(IBMAEstimator): "p" P-value map from one-sample test. "est" Fixed effects estimate for intercept test. "se" Standard error of fixed effects estimate. + "dof" Degrees of freedom map from one-sample test. ============== =============================================================================== Warnings @@ -358,9 +463,11 @@ class WeightedLeastSquares(IBMAEstimator): will likely result in biased results. The extent of this bias is currently unknown. - All image-based meta-analysis estimators adopt an aggressive masking + By default, all image-based meta-analysis estimators adopt an aggressive masking strategy, in which any voxels with a value of zero in any of the input maps - will be removed from the analysis. + will be removed from the analysis. Setting ``aggressive_mask=False`` will + instead run tha analysis in bags of voxels that have a valid value across + the same studies. References ---------- @@ -398,22 +505,51 @@ def _fit(self, dataset): "with this Estimator." ) - pymare_dset = pymare.Dataset(y=self.inputs_["beta_maps"], v=self.inputs_["varcope_maps"]) - est = pymare.estimators.WeightedLeastSquares(tau2=self.tau2) - est.fit_dataset(pymare_dset) - est_summary = est.summary() + if self.aggressive_mask: + pymare_dset = pymare.Dataset( + y=self.inputs_["beta_maps"], v=self.inputs_["varcope_maps"] + ) + est = pymare.estimators.WeightedLeastSquares(tau2=self.tau2) + est.fit_dataset(pymare_dset) + est_summary = est.summary() + + fe_stats = est_summary.get_fe_stats() + z_map = _boolean_unmask(fe_stats["z"].squeeze(), self.inputs_["aggressive_mask"]) + p_map = _boolean_unmask(fe_stats["p"].squeeze(), self.inputs_["aggressive_mask"]) + est_map = _boolean_unmask(fe_stats["est"].squeeze(), self.inputs_["aggressive_mask"]) + se_map = _boolean_unmask(fe_stats["se"].squeeze(), self.inputs_["aggressive_mask"]) + dof_map = np.tile( + self.inputs_["beta_maps"].shape[0] - 1, self.inputs_["beta_maps"].shape[1] + ).astype(np.int32) + dof_map = _boolean_unmask(dof_map, self.inputs_["aggressive_mask"]) - fe_stats = est_summary.get_fe_stats() - # tau2 is an float, not a map, so it can't go in the results dictionary - maps = { - "z": _boolean_unmask(fe_stats["z"].squeeze(), self.inputs_["aggressive_mask"]), - "p": _boolean_unmask(fe_stats["p"].squeeze(), self.inputs_["aggressive_mask"]), - "est": _boolean_unmask(fe_stats["est"].squeeze(), self.inputs_["aggressive_mask"]), - "se": _boolean_unmask(fe_stats["se"].squeeze(), self.inputs_["aggressive_mask"]), - } + else: + n_total_voxels = self.inputs_["beta_maps"].shape[1] + z_map = np.zeros(n_total_voxels, dtype=float) + p_map = np.zeros(n_total_voxels, dtype=float) + est_map = np.zeros(n_total_voxels, dtype=float) + se_map = np.zeros(n_total_voxels, dtype=float) + dof_map = np.zeros(n_total_voxels, dtype=np.int32) + beta_bags = self.inputs_["data_bags"]["beta_maps"] + varcope_bags = self.inputs_["data_bags"]["varcope_maps"] + for beta_bag, varcope_bag in zip(beta_bags, varcope_bags): + pymare_dset = pymare.Dataset(y=beta_bag["values"], v=varcope_bag["values"]) + est = pymare.estimators.WeightedLeastSquares(tau2=self.tau2) + est.fit_dataset(pymare_dset) + est_summary = est.summary() + + fe_stats = est_summary.get_fe_stats() + z_map[beta_bag["voxel_mask"]] = fe_stats["z"].squeeze() + p_map[beta_bag["voxel_mask"]] = fe_stats["p"].squeeze() + est_map[beta_bag["voxel_mask"]] = fe_stats["est"].squeeze() + se_map[beta_bag["voxel_mask"]] = fe_stats["se"].squeeze() + dof_map[beta_bag["voxel_mask"]] = beta_bag["values"].shape[0] - 1 + + # tau2 is a float, not a map, so it can't go into the results dictionary tables = { "level-estimator": pd.DataFrame(columns=["tau2"], data=[self.tau2]), } + maps = {"z": z_map, "p": p_map, "est": est_map, "se": se_map, "dof": dof_map} description = self._generate_description() return maps, tables, description @@ -422,6 +558,10 @@ def _fit(self, dataset): class DerSimonianLaird(IBMAEstimator): """DerSimonian-Laird meta-regression estimator. + .. versionchanged:: 0.2.1 + + * New parameter: ``aggressive_mask``, to control whether to use an aggressive mask. + .. versionchanged:: 0.0.12 * Add "se" to outputs. @@ -435,6 +575,15 @@ class DerSimonianLaird(IBMAEstimator): Estimates the between-subject variance tau^2 using the :footcite:t:`dersimonian1986meta` method-of-moments approach :footcite:p:`dersimonian1986meta,kosmidis2017improving`. + Parameters + ---------- + aggressive_mask : :obj:`bool`, optional + Voxels with a value of zero of NaN in any of the input maps will be removed + from the analysis. + If False, all voxels are included by running a separate analysis on bags + of voxels that belong that have a valid value across the same studies. + Default is True. + Notes ----- Requires :term:`beta` and :term:`varcope` images. @@ -447,6 +596,7 @@ class DerSimonianLaird(IBMAEstimator): "est" Fixed effects estimate for intercept test. "se" Standard error of fixed effects estimate. "tau2" Estimated between-study variance. + "dof" Degrees of freedom map from one-sample test. ============== =============================================================================== Warnings @@ -455,9 +605,11 @@ class DerSimonianLaird(IBMAEstimator): will likely result in biased results. The extent of this bias is currently unknown. - All image-based meta-analysis estimators adopt an aggressive masking + By default, all image-based meta-analysis estimators adopt an aggressive masking strategy, in which any voxels with a value of zero in any of the input maps - will be removed from the analysis. + will be removed from the analysis. Setting ``aggressive_mask=False`` will + instead run tha analysis in bags of voxels that have a valid value across + the same studies. References ---------- @@ -492,20 +644,57 @@ def _fit(self, dataset): "with this Estimator." ) - est = pymare.estimators.DerSimonianLaird() - pymare_dset = pymare.Dataset(y=self.inputs_["beta_maps"], v=self.inputs_["varcope_maps"]) - est.fit_dataset(pymare_dset) - est_summary = est.summary() + if self.aggressive_mask: + est = pymare.estimators.DerSimonianLaird() + pymare_dset = pymare.Dataset( + y=self.inputs_["beta_maps"], v=self.inputs_["varcope_maps"] + ) + est.fit_dataset(pymare_dset) + est_summary = est.summary() + + fe_stats = est_summary.get_fe_stats() + z_map = _boolean_unmask(fe_stats["z"].squeeze(), self.inputs_["aggressive_mask"]) + p_map = _boolean_unmask(fe_stats["p"].squeeze(), self.inputs_["aggressive_mask"]) + est_map = _boolean_unmask(fe_stats["est"].squeeze(), self.inputs_["aggressive_mask"]) + se_map = _boolean_unmask(fe_stats["se"].squeeze(), self.inputs_["aggressive_mask"]) + tau2_map = _boolean_unmask(est_summary.tau2.squeeze(), self.inputs_["aggressive_mask"]) + dof_map = np.tile( + self.inputs_["beta_maps"].shape[0] - 1, self.inputs_["beta_maps"].shape[1] + ).astype(np.int32) + dof_map = _boolean_unmask(dof_map, self.inputs_["aggressive_mask"]) + + else: + n_total_voxels = self.inputs_["beta_maps"].shape[1] + z_map = np.zeros(n_total_voxels, dtype=float) + p_map = np.zeros(n_total_voxels, dtype=float) + est_map = np.zeros(n_total_voxels, dtype=float) + se_map = np.zeros(n_total_voxels, dtype=float) + tau2_map = np.zeros(n_total_voxels, dtype=float) + dof_map = np.zeros(n_total_voxels, dtype=np.int32) + beta_bags = self.inputs_["data_bags"]["beta_maps"] + varcope_bags = self.inputs_["data_bags"]["varcope_maps"] + for beta_bag, varcope_bag in zip(beta_bags, varcope_bags): + est = pymare.estimators.DerSimonianLaird() + pymare_dset = pymare.Dataset(y=beta_bag["values"], v=varcope_bag["values"]) + est.fit_dataset(pymare_dset) + est_summary = est.summary() + + fe_stats = est_summary.get_fe_stats() + z_map[beta_bag["voxel_mask"]] = fe_stats["z"].squeeze() + p_map[beta_bag["voxel_mask"]] = fe_stats["p"].squeeze() + est_map[beta_bag["voxel_mask"]] = fe_stats["est"].squeeze() + se_map[beta_bag["voxel_mask"]] = fe_stats["se"].squeeze() + tau2_map[beta_bag["voxel_mask"]] = est_summary.tau2.squeeze() + dof_map[beta_bag["voxel_mask"]] = beta_bag["values"].shape[0] - 1 - fe_stats = est_summary.get_fe_stats() maps = { - "z": _boolean_unmask(fe_stats["z"].squeeze(), self.inputs_["aggressive_mask"]), - "p": _boolean_unmask(fe_stats["p"].squeeze(), self.inputs_["aggressive_mask"]), - "est": _boolean_unmask(fe_stats["est"].squeeze(), self.inputs_["aggressive_mask"]), - "se": _boolean_unmask(fe_stats["se"].squeeze(), self.inputs_["aggressive_mask"]), - "tau2": _boolean_unmask(est_summary.tau2.squeeze(), self.inputs_["aggressive_mask"]), + "z": z_map, + "p": p_map, + "est": est_map, + "se": se_map, + "tau2": tau2_map, + "dof": dof_map, } - description = self._generate_description() return maps, {}, description @@ -514,6 +703,10 @@ def _fit(self, dataset): class Hedges(IBMAEstimator): """Hedges meta-regression estimator. + .. versionchanged:: 0.2.1 + + * New parameter: ``aggressive_mask``, to control whether to use an aggressive mask. + .. versionchanged:: 0.0.12 * Add "se" to outputs. @@ -527,6 +720,15 @@ class Hedges(IBMAEstimator): Estimates the between-subject variance tau^2 using the :footcite:t:`hedges2014statistical` approach. + Parameters + ---------- + aggressive_mask : :obj:`bool`, optional + Voxels with a value of zero of NaN in any of the input maps will be removed + from the analysis. + If False, all voxels are included by running a separate analysis on bags + of voxels that belong that have a valid value across the same studies. + Default is True. + Notes ----- Requires :term:`beta` and :term:`varcope` images. @@ -539,6 +741,7 @@ class Hedges(IBMAEstimator): "est" Fixed effects estimate for intercept test. "se" Standard error of fixed effects estimate. "tau2" Estimated between-study variance. + "dof" Degrees of freedom map from one-sample test. ============== =============================================================================== Warnings @@ -547,9 +750,11 @@ class Hedges(IBMAEstimator): will likely result in biased results. The extent of this bias is currently unknown. - All image-based meta-analysis estimators adopt an aggressive masking + By default, all image-based meta-analysis estimators adopt an aggressive masking strategy, in which any voxels with a value of zero in any of the input maps - will be removed from the analysis. + will be removed from the analysis. Setting ``aggressive_mask=False`` will + instead run tha analysis in bags of voxels that have a valid value across + the same studies. References ---------- @@ -583,17 +788,56 @@ def _fit(self, dataset): "with this Estimator." ) - est = pymare.estimators.Hedges() - pymare_dset = pymare.Dataset(y=self.inputs_["beta_maps"], v=self.inputs_["varcope_maps"]) - est.fit_dataset(pymare_dset) - est_summary = est.summary() - fe_stats = est_summary.get_fe_stats() + if self.aggressive_mask: + est = pymare.estimators.Hedges() + pymare_dset = pymare.Dataset( + y=self.inputs_["beta_maps"], v=self.inputs_["varcope_maps"] + ) + est.fit_dataset(pymare_dset) + est_summary = est.summary() + fe_stats = est_summary.get_fe_stats() + + z_map = _boolean_unmask(fe_stats["z"].squeeze(), self.inputs_["aggressive_mask"]) + p_map = _boolean_unmask(fe_stats["p"].squeeze(), self.inputs_["aggressive_mask"]) + est_map = _boolean_unmask(fe_stats["est"].squeeze(), self.inputs_["aggressive_mask"]) + se_map = _boolean_unmask(fe_stats["se"].squeeze(), self.inputs_["aggressive_mask"]) + tau2_map = _boolean_unmask(est_summary.tau2.squeeze(), self.inputs_["aggressive_mask"]) + dof_map = np.tile( + self.inputs_["beta_maps"].shape[0] - 1, self.inputs_["beta_maps"].shape[1] + ).astype(np.int32) + dof_map = _boolean_unmask(dof_map, self.inputs_["aggressive_mask"]) + + else: + n_total_voxels = self.inputs_["beta_maps"].shape[1] + z_map = np.zeros(n_total_voxels, dtype=float) + p_map = np.zeros(n_total_voxels, dtype=float) + est_map = np.zeros(n_total_voxels, dtype=float) + se_map = np.zeros(n_total_voxels, dtype=float) + tau2_map = np.zeros(n_total_voxels, dtype=float) + dof_map = np.zeros(n_total_voxels, dtype=np.int32) + beta_bags = self.inputs_["data_bags"]["beta_maps"] + varcope_bags = self.inputs_["data_bags"]["varcope_maps"] + for beta_bag, varcope_bag in zip(beta_bags, varcope_bags): + est = pymare.estimators.Hedges() + pymare_dset = pymare.Dataset(y=beta_bag["values"], v=varcope_bag["values"]) + est.fit_dataset(pymare_dset) + est_summary = est.summary() + fe_stats = est_summary.get_fe_stats() + + z_map[beta_bag["voxel_mask"]] = fe_stats["z"].squeeze() + p_map[beta_bag["voxel_mask"]] = fe_stats["p"].squeeze() + est_map[beta_bag["voxel_mask"]] = fe_stats["est"].squeeze() + se_map[beta_bag["voxel_mask"]] = fe_stats["se"].squeeze() + tau2_map[beta_bag["voxel_mask"]] = est_summary.tau2.squeeze() + dof_map[beta_bag["voxel_mask"]] = beta_bag["values"].shape[0] - 1 + maps = { - "z": _boolean_unmask(fe_stats["z"].squeeze(), self.inputs_["aggressive_mask"]), - "p": _boolean_unmask(fe_stats["p"].squeeze(), self.inputs_["aggressive_mask"]), - "est": _boolean_unmask(fe_stats["est"].squeeze(), self.inputs_["aggressive_mask"]), - "se": _boolean_unmask(fe_stats["se"].squeeze(), self.inputs_["aggressive_mask"]), - "tau2": _boolean_unmask(est_summary.tau2.squeeze(), self.inputs_["aggressive_mask"]), + "z": z_map, + "p": p_map, + "est": est_map, + "se": se_map, + "tau2": tau2_map, + "dof": dof_map, } description = self._generate_description() @@ -603,6 +847,10 @@ def _fit(self, dataset): class SampleSizeBasedLikelihood(IBMAEstimator): """Method estimates with known sample sizes but unknown sampling variances. + .. versionchanged:: 0.2.1 + + * New parameter: ``aggressive_mask``, to control whether to use an aggressive mask. + .. versionchanged:: 0.0.12 * Add "se" and "sigma2" to outputs. @@ -618,6 +866,12 @@ class SampleSizeBasedLikelihood(IBMAEstimator): Parameters ---------- + aggressive_mask : :obj:`bool`, optional + Voxels with a value of zero of NaN in any of the input maps will be removed + from the analysis. + If False, all voxels are included by running a separate analysis on bags + of voxels that belong that have a valid value across the same studies. + Default is True. method : {'ml', 'reml'}, optional The estimation method to use. The available options are @@ -639,6 +893,7 @@ class SampleSizeBasedLikelihood(IBMAEstimator): "se" Standard error of fixed effects estimate. "tau2" Estimated between-study variance. "sigma2" Estimated within-study variance. Assumed to be the same for all studies. + "dof" Degrees of freedom map from one-sample test. ============== =============================================================================== Homogeneity of sigma^2 across studies is assumed. @@ -652,9 +907,11 @@ class SampleSizeBasedLikelihood(IBMAEstimator): method should not be used on full brains, unless you can submit your code to a job scheduler. - All image-based meta-analysis estimators adopt an aggressive masking + By default, all image-based meta-analysis estimators adopt an aggressive masking strategy, in which any voxels with a value of zero in any of the input maps - will be removed from the analysis. + will be removed from the analysis. Setting ``aggressive_mask=False`` will + instead run tha analysis in bags of voxels that have a valid value across + the same studies. See Also -------- @@ -685,23 +942,66 @@ def _fit(self, dataset): self.dataset = dataset self.masker = self.masker or dataset.masker - sample_sizes = np.array([np.mean(n) for n in self.inputs_["sample_sizes"]]) - n_maps = np.tile(sample_sizes, (self.inputs_["beta_maps"].shape[1], 1)).T - pymare_dset = pymare.Dataset(y=self.inputs_["beta_maps"], n=n_maps) - est = pymare.estimators.SampleSizeBasedLikelihoodEstimator(method=self.method) - est.fit_dataset(pymare_dset) - est_summary = est.summary() - fe_stats = est_summary.get_fe_stats() + if self.aggressive_mask: + sample_sizes = np.array([np.mean(n) for n in self.inputs_["sample_sizes"]]) + n_maps = np.tile(sample_sizes, (self.inputs_["beta_maps"].shape[1], 1)).T + + pymare_dset = pymare.Dataset(y=self.inputs_["beta_maps"], n=n_maps) + est = pymare.estimators.SampleSizeBasedLikelihoodEstimator(method=self.method) + est.fit_dataset(pymare_dset) + est_summary = est.summary() + fe_stats = est_summary.get_fe_stats() + + z_map = _boolean_unmask(fe_stats["z"].squeeze(), self.inputs_["aggressive_mask"]) + p_map = _boolean_unmask(fe_stats["p"].squeeze(), self.inputs_["aggressive_mask"]) + est_map = _boolean_unmask(fe_stats["est"].squeeze(), self.inputs_["aggressive_mask"]) + se_map = _boolean_unmask(fe_stats["se"].squeeze(), self.inputs_["aggressive_mask"]) + tau2_map = _boolean_unmask(est_summary.tau2.squeeze(), self.inputs_["aggressive_mask"]) + sigma2_map = _boolean_unmask( + est.params_["sigma2"].squeeze(), self.inputs_["aggressive_mask"] + ) + dof_map = np.tile( + self.inputs_["beta_maps"].shape[0] - 1, self.inputs_["beta_maps"].shape[1] + ).astype(np.int32) + dof_map = _boolean_unmask(dof_map, self.inputs_["aggressive_mask"]) + + else: + n_total_voxels = self.inputs_["beta_maps"].shape[1] + z_map = np.zeros(n_total_voxels, dtype=float) + p_map = np.zeros(n_total_voxels, dtype=float) + est_map = np.zeros(n_total_voxels, dtype=float) + se_map = np.zeros(n_total_voxels, dtype=float) + tau2_map = np.zeros(n_total_voxels, dtype=float) + sigma2_map = np.zeros(n_total_voxels, dtype=float) + dof_map = np.zeros(n_total_voxels, dtype=np.int32) + for bag in self.inputs_["data_bags"]["beta_maps"]: + study_mask = bag["study_mask"] + sample_sizes_ex = [self.inputs_["sample_sizes"][study] for study in study_mask] + sample_sizes = np.array([np.mean(n) for n in sample_sizes_ex]) + n_maps = np.tile(sample_sizes, (bag["values"].shape[1], 1)).T + + pymare_dset = pymare.Dataset(y=bag["values"], n=n_maps) + est = pymare.estimators.SampleSizeBasedLikelihoodEstimator(method=self.method) + est.fit_dataset(pymare_dset) + est_summary = est.summary() + fe_stats = est_summary.get_fe_stats() + + z_map[bag["voxel_mask"]] = fe_stats["z"].squeeze() + p_map[bag["voxel_mask"]] = fe_stats["p"].squeeze() + est_map[bag["voxel_mask"]] = fe_stats["est"].squeeze() + se_map[bag["voxel_mask"]] = fe_stats["se"].squeeze() + tau2_map[bag["voxel_mask"]] = est_summary.tau2.squeeze() + sigma2_map[bag["voxel_mask"]] = est.params_["sigma2"].squeeze() + dof_map[bag["voxel_mask"]] = bag["values"].shape[0] - 1 + maps = { - "z": _boolean_unmask(fe_stats["z"].squeeze(), self.inputs_["aggressive_mask"]), - "p": _boolean_unmask(fe_stats["p"].squeeze(), self.inputs_["aggressive_mask"]), - "est": _boolean_unmask(fe_stats["est"].squeeze(), self.inputs_["aggressive_mask"]), - "se": _boolean_unmask(fe_stats["se"].squeeze(), self.inputs_["aggressive_mask"]), - "tau2": _boolean_unmask(est_summary.tau2.squeeze(), self.inputs_["aggressive_mask"]), - "sigma2": _boolean_unmask( - est.params_["sigma2"].squeeze(), - self.inputs_["aggressive_mask"], - ), + "z": z_map, + "p": p_map, + "est": est_map, + "se": se_map, + "tau2": tau2_map, + "sigma2": sigma2_map, + "dof": dof_map, } description = self._generate_description() @@ -711,6 +1011,10 @@ def _fit(self, dataset): class VarianceBasedLikelihood(IBMAEstimator): """A likelihood-based meta-analysis method for estimates with known variances. + .. versionchanged:: 0.2.1 + + * New parameter: ``aggressive_mask``, to control whether to use an aggressive mask. + .. versionchanged:: 0.0.12 Add "se" output. @@ -727,6 +1031,12 @@ class VarianceBasedLikelihood(IBMAEstimator): Parameters ---------- + aggressive_mask : :obj:`bool`, optional + Voxels with a value of zero of NaN in any of the input maps will be removed + from the analysis. + If False, all voxels are included by running a separate analysis on bags + of voxels that belong that have a valid value across the same studies. + Default is True. method : {'ml', 'reml'}, optional The estimation method to use. The available options are @@ -747,6 +1057,7 @@ class VarianceBasedLikelihood(IBMAEstimator): "est" Fixed effects estimate for intercept test. "se" Standard error of fixed effects estimate. "tau2" Estimated between-study variance. + "dof" Degrees of freedom map from one-sample test. ============== =============================================================================== The ML and REML solutions are obtained via SciPy's scalar function @@ -763,9 +1074,11 @@ class VarianceBasedLikelihood(IBMAEstimator): will likely result in biased results. The extent of this bias is currently unknown. - All image-based meta-analysis estimators adopt an aggressive masking + By default, all image-based meta-analysis estimators adopt an aggressive masking strategy, in which any voxels with a value of zero in any of the input maps - will be removed from the analysis. + will be removed from the analysis. Setting ``aggressive_mask=False`` will + instead run tha analysis in bags of voxels that have a valid value across + the same studies. References ---------- @@ -804,18 +1117,57 @@ def _fit(self, dataset): "with this Estimator." ) - est = pymare.estimators.VarianceBasedLikelihoodEstimator(method=self.method) + if self.aggressive_mask: + est = pymare.estimators.VarianceBasedLikelihoodEstimator(method=self.method) + + pymare_dset = pymare.Dataset( + y=self.inputs_["beta_maps"], v=self.inputs_["varcope_maps"] + ) + est.fit_dataset(pymare_dset) + est_summary = est.summary() + fe_stats = est_summary.get_fe_stats() + + z_map = _boolean_unmask(fe_stats["z"].squeeze(), self.inputs_["aggressive_mask"]) + p_map = _boolean_unmask(fe_stats["p"].squeeze(), self.inputs_["aggressive_mask"]) + est_map = _boolean_unmask(fe_stats["est"].squeeze(), self.inputs_["aggressive_mask"]) + se_map = _boolean_unmask(fe_stats["se"].squeeze(), self.inputs_["aggressive_mask"]) + tau2_map = _boolean_unmask(est_summary.tau2.squeeze(), self.inputs_["aggressive_mask"]) + dof_map = np.tile( + self.inputs_["beta_maps"].shape[0] - 1, self.inputs_["beta_maps"].shape[1] + ).astype(np.int32) + dof_map = _boolean_unmask(dof_map, self.inputs_["aggressive_mask"]) + else: + n_total_voxels = self.inputs_["beta_maps"].shape[1] + z_map = np.zeros(n_total_voxels, dtype=float) + p_map = np.zeros(n_total_voxels, dtype=float) + est_map = np.zeros(n_total_voxels, dtype=float) + se_map = np.zeros(n_total_voxels, dtype=float) + tau2_map = np.zeros(n_total_voxels, dtype=float) + dof_map = np.zeros(n_total_voxels, dtype=np.int32) + beta_bags = self.inputs_["data_bags"]["beta_maps"] + varcope_bags = self.inputs_["data_bags"]["varcope_maps"] + for beta_bag, varcope_bag in zip(beta_bags, varcope_bags): + est = pymare.estimators.VarianceBasedLikelihoodEstimator(method=self.method) + + pymare_dset = pymare.Dataset(y=beta_bag["values"], v=varcope_bag["values"]) + est.fit_dataset(pymare_dset) + est_summary = est.summary() + fe_stats = est_summary.get_fe_stats() + + z_map[beta_bag["voxel_mask"]] = fe_stats["z"].squeeze() + p_map[beta_bag["voxel_mask"]] = fe_stats["p"].squeeze() + est_map[beta_bag["voxel_mask"]] = fe_stats["est"].squeeze() + se_map[beta_bag["voxel_mask"]] = fe_stats["se"].squeeze() + tau2_map[beta_bag["voxel_mask"]] = est_summary.tau2.squeeze() + dof_map[beta_bag["voxel_mask"]] = beta_bag["values"].shape[0] - 1 - pymare_dset = pymare.Dataset(y=self.inputs_["beta_maps"], v=self.inputs_["varcope_maps"]) - est.fit_dataset(pymare_dset) - est_summary = est.summary() - fe_stats = est_summary.get_fe_stats() maps = { - "z": _boolean_unmask(fe_stats["z"].squeeze(), self.inputs_["aggressive_mask"]), - "p": _boolean_unmask(fe_stats["p"].squeeze(), self.inputs_["aggressive_mask"]), - "est": _boolean_unmask(fe_stats["est"].squeeze(), self.inputs_["aggressive_mask"]), - "se": _boolean_unmask(fe_stats["se"].squeeze(), self.inputs_["aggressive_mask"]), - "tau2": _boolean_unmask(est_summary.tau2.squeeze(), self.inputs_["aggressive_mask"]), + "z": z_map, + "p": p_map, + "est": est_map, + "se": se_map, + "tau2": tau2_map, + "dof": dof_map, } description = self._generate_description() @@ -825,6 +1177,10 @@ def _fit(self, dataset): class PermutedOLS(IBMAEstimator): r"""An analysis with permuted ordinary least squares (OLS), using nilearn. + .. versionchanged:: 0.2.1 + + * New parameter: ``aggressive_mask``, to control whether to use an aggressive mask. + .. versionchanged:: 0.0.12 * Use beta maps instead of z maps. @@ -839,6 +1195,12 @@ class PermutedOLS(IBMAEstimator): Parameters ---------- + aggressive_mask : :obj:`bool`, optional + Voxels with a value of zero of NaN in any of the input maps will be removed + from the analysis. + If False, all voxels are included by running a separate analysis on bags + of voxels that belong that have a valid value across the same studies. + Default is True. two_sided : :obj:`bool`, optional If True, performs an unsigned t-test. Both positive and negative effects are considered; the null hypothesis is that the effect is zero. If False, only positive effects are @@ -854,15 +1216,18 @@ class PermutedOLS(IBMAEstimator): ============== =============================================================================== "t" T-statistic map from one-sample test. "z" Z-statistic map from one-sample test. + "dof" Degrees of freedom map from one-sample test. ============== =============================================================================== Available correction methods: :func:`PermutedOLS.correct_fwe_montecarlo` Warnings -------- - All image-based meta-analysis estimators adopt an aggressive masking + By default, all image-based meta-analysis estimators adopt an aggressive masking strategy, in which any voxels with a value of zero in any of the input maps - will be removed from the analysis. + will be removed from the analysis. Setting ``aggressive_mask=False`` will + instead run tha analysis in bags of voxels that have a valid value across + the same studies. References ---------- @@ -891,29 +1256,66 @@ def _generate_description(self): def _fit(self, dataset): self.dataset = dataset - # Use intercept as explanatory variable - self.parameters_["tested_vars"] = np.ones((self.inputs_["beta_maps"].shape[0], 1)) - self.parameters_["confounding_vars"] = None - - _, t_map, _ = permuted_ols( - self.parameters_["tested_vars"], - self.inputs_["beta_maps"], - confounding_vars=self.parameters_["confounding_vars"], - model_intercept=False, # modeled by tested_vars - n_perm=0, - two_sided_test=self.two_sided, - random_state=42, - n_jobs=1, - verbose=0, - ) - # Convert t to z, preserving signs - dof = self.parameters_["tested_vars"].shape[0] - self.parameters_["tested_vars"].shape[1] - z_map = t_to_z(t_map, dof) - maps = { - "t": _boolean_unmask(t_map.squeeze(), self.inputs_["aggressive_mask"]), - "z": _boolean_unmask(z_map.squeeze(), self.inputs_["aggressive_mask"]), - } + if self.aggressive_mask: + # Use intercept as explanatory variable + self.parameters_["tested_vars"] = np.ones((self.inputs_["beta_maps"].shape[0], 1)) + self.parameters_["confounding_vars"] = None + + _, t_map, _ = permuted_ols( + self.parameters_["tested_vars"], + self.inputs_["beta_maps"], + confounding_vars=self.parameters_["confounding_vars"], + model_intercept=False, # modeled by tested_vars + n_perm=0, + two_sided_test=self.two_sided, + random_state=42, + n_jobs=1, + verbose=0, + ) + + # Convert t to z, preserving signs + dof = ( + self.parameters_["tested_vars"].shape[0] - self.parameters_["tested_vars"].shape[1] + ) + z_map = t_to_z(t_map, dof) + + t_map = _boolean_unmask(t_map.squeeze(), self.inputs_["aggressive_mask"]) + z_map = _boolean_unmask(z_map.squeeze(), self.inputs_["aggressive_mask"]) + dof_map = np.tile(dof, self.inputs_["beta_maps"].shape[1]).astype(np.int32) + dof_map = _boolean_unmask(dof_map, self.inputs_["aggressive_mask"]) + + else: + n_total_voxels = self.inputs_["beta_maps"].shape[1] + t_map = np.zeros(n_total_voxels, dtype=float) + z_map = np.zeros(n_total_voxels, dtype=float) + dof_map = np.zeros(n_total_voxels, dtype=np.int32) + for bag in self.inputs_["data_bags"]["beta_maps"]: + # Use intercept as explanatory variable + bag["tested_vars"] = np.ones((bag["values"].shape[0], 1)) + bag["confounding_vars"] = None + + _, t_map_tmp, _ = permuted_ols( + bag["tested_vars"], + bag["values"], + confounding_vars=bag["confounding_vars"], + model_intercept=False, # modeled by tested_vars + n_perm=0, + two_sided_test=self.two_sided, + random_state=42, + n_jobs=1, + verbose=0, + ) + + # Convert t to z, preserving signs + dof = bag["tested_vars"].shape[0] - bag["tested_vars"].shape[1] + z_map_tmp = t_to_z(t_map_tmp, dof) + + t_map[bag["voxel_mask"]] = t_map_tmp.squeeze() + z_map[bag["voxel_mask"]] = z_map_tmp.squeeze() + dof_map[bag["voxel_mask"]] = dof + + maps = {"t": t_map, "z": z_map, "dof": dof_map} description = self._generate_description() return maps, {}, description @@ -962,32 +1364,59 @@ def correct_fwe_montecarlo(self, result, n_iters=10000, n_cores=1): """ n_cores = _check_ncores(n_cores) - log_p_map, t_map, _ = permuted_ols( - self.parameters_["tested_vars"], - self.inputs_["beta_maps"], - confounding_vars=self.parameters_["confounding_vars"], - model_intercept=False, # modeled by tested_vars - n_perm=n_iters, - two_sided_test=self.two_sided, - random_state=42, - n_jobs=n_cores, - verbose=0, - ) + if self.aggressive_mask: + log_p_map, t_map, _ = permuted_ols( + self.parameters_["tested_vars"], + self.inputs_["beta_maps"], + confounding_vars=self.parameters_["confounding_vars"], + model_intercept=False, # modeled by tested_vars + n_perm=n_iters, + two_sided_test=self.two_sided, + random_state=42, + n_jobs=n_cores, + verbose=0, + ) - # Fill complete maps - p_map = np.power(10.0, -log_p_map) + # Fill complete maps + p_map = np.power(10.0, -log_p_map) - # Convert p to z, preserving signs - sign = np.sign(t_map) - sign[sign == 0] = 1 - z_map = p_to_z(p_map, tail="two") * sign - maps = { - "logp_level-voxel": _boolean_unmask( - log_p_map.squeeze(), self.inputs_["aggressive_mask"] - ), - "z_level-voxel": _boolean_unmask(z_map.squeeze(), self.inputs_["aggressive_mask"]), - } + # Convert p to z, preserving signs + sign = np.sign(t_map) + sign[sign == 0] = 1 + z_map = p_to_z(p_map, tail="two") * sign + + log_p_map = _boolean_unmask(log_p_map.squeeze(), self.inputs_["aggressive_mask"]) + z_map = _boolean_unmask(z_map.squeeze(), self.inputs_["aggressive_mask"]) + + else: + n_total_voxels = self.inputs_["beta_maps"].shape[1] + log_p_map = np.zeros(n_total_voxels, dtype=float) + z_map = np.zeros(n_total_voxels, dtype=float) + for bag in self.inputs_["data_bags"]["beta_maps"]: + log_p_map_tmp, t_map_tmp, _ = permuted_ols( + bag["tested_vars"], + bag["values"], + confounding_vars=bag["confounding_vars"], + model_intercept=False, # modeled by tested_vars + n_perm=n_iters, + two_sided_test=self.two_sided, + random_state=42, + n_jobs=n_cores, + verbose=0, + ) + + # Fill complete maps + p_map_tmp = np.power(10.0, -log_p_map_tmp) + + # Convert p to z, preserving signs + sign = np.sign(t_map_tmp) + sign[sign == 0] = 1 + z_map_tmp = p_to_z(p_map_tmp, tail="two") * sign + + log_p_map[bag["voxel_mask"]] = log_p_map_tmp.squeeze() + z_map[bag["voxel_mask"]] = z_map_tmp.squeeze() + maps = {"logp_level-voxel": log_p_map, "z_level-voxel": z_map} description = ( "Family-wise error rate correction was performed using Nilearn's " "\\citep{10.3389/fninf.2014.00014} permuted OLS method, in which null distributions " diff --git a/nimare/meta/utils.py b/nimare/meta/utils.py index 86b3281a0..262bb519c 100755 --- a/nimare/meta/utils.py +++ b/nimare/meta/utils.py @@ -3,6 +3,7 @@ import numpy as np import sparse +from numba import jit from scipy import ndimage from nimare.utils import unique_rows @@ -384,3 +385,80 @@ def _calculate_cluster_measures(arr3d, threshold, conn, tail="upper"): max_size = 0 return max_size, max_mass + + +@jit(nopython=True, cache=True) +def _apply_liberal_mask(data): + """Separate input image data in bags of voxels that have a valid value across the same studies. + + Parameters + ---------- + data : (S x V) :class:`numpy.ndarray` + 2D numpy array (S x V) of images, where S is study and V is voxel. + + Returns + ------- + values_lst : :obj:`list` of :obj:`numpy.ndarray` + List of 2D numpy arrays (s x v) of images, where the voxel v have a valid + value in study s. + voxel_mask_lst : :obj:`list` of :obj:`numpy.ndarray` + List of 1D numpy arrays (v) of voxel indices for the corresponding bag. + study_mask_lst : :obj:`list` of :obj:`numpy.ndarray` + List of 1D numpy arrays (s) of study indices for the corresponding bag. + + Notes + ----- + Parts of the function are implemented with nested for loops to + improve the speed with the numba compiler. + + """ + MIN_STUDY_THRESH = 2 + + n_voxels = data.shape[1] + # Get indices of non-nan and zero value of studies for each voxel + mask = ~np.isnan(data) & (data != 0) + study_by_voxels_idxs = [np.where(mask[:, i])[0] for i in range(n_voxels)] + + # Group studies by the same number of non-nan voxels + matches = [] + all_indices = [] + for col_i in range(n_voxels): + if col_i in all_indices: + continue + + vox_match = [col_i] + all_indices.append(col_i) + for col_j in range(col_i + 1, n_voxels): + if ( + len(study_by_voxels_idxs[col_i]) == len(study_by_voxels_idxs[col_j]) + and np.array_equal(study_by_voxels_idxs[col_i], study_by_voxels_idxs[col_j]) + and col_j not in all_indices + ): + vox_match.append(col_j) + all_indices.append(col_j) + + matches.append(np.array(vox_match)) + + values_lst, voxel_mask_lst, study_mask_lst = [], [], [] + for voxel_mask in matches: + n_masked_voxels = len(voxel_mask) + # This is the same for all voxels in the match + study_mask = study_by_voxels_idxs[voxel_mask[0]] + + if len(study_mask) < MIN_STUDY_THRESH: + # TODO: Figure out how raise a warning in numba + # warnings.warn( + # f"Removing voxels: {voxel_mask} from the analysis. Not present in 2+ studies." + # ) + continue + + values = np.zeros((len(study_mask), n_masked_voxels)) + for vox_i, vox in enumerate(voxel_mask): + for std_i, study in enumerate(study_mask): + values[std_i, vox_i] = data[study, vox] + + values_lst.append(values) + voxel_mask_lst.append(voxel_mask) + study_mask_lst.append(study_mask) + + return values_lst, voxel_mask_lst, study_mask_lst diff --git a/nimare/tests/test_meta_ibma.py b/nimare/tests/test_meta_ibma.py index 468396308..2597ab46c 100644 --- a/nimare/tests/test_meta_ibma.py +++ b/nimare/tests/test_meta_ibma.py @@ -20,7 +20,7 @@ {}, FDRCorrector, {"method": "indep", "alpha": 0.001}, - ("z", "p"), + ("z", "p", "dof"), id="Fishers", ), pytest.param( @@ -28,7 +28,7 @@ {"use_sample_size": False}, None, {}, - ("z", "p"), + ("z", "p", "dof"), id="Stouffers", ), pytest.param( @@ -36,7 +36,7 @@ {"use_sample_size": True}, None, {}, - ("z", "p"), + ("z", "p", "dof"), id="Stouffers_weighted", ), pytest.param( @@ -44,7 +44,7 @@ {"tau2": 0}, None, {}, - ("z", "p", "est", "se"), + ("z", "p", "est", "se", "dof"), id="WeightedLeastSquares", ), pytest.param( @@ -52,7 +52,7 @@ {}, None, {}, - ("z", "p", "est", "se", "tau2"), + ("z", "p", "est", "se", "tau2", "dof"), id="DerSimonianLaird", ), pytest.param( @@ -60,7 +60,7 @@ {}, None, {}, - ("z", "p", "est", "se", "tau2"), + ("z", "p", "est", "se", "tau2", "dof"), id="Hedges", ), pytest.param( @@ -68,7 +68,7 @@ {"method": "ml"}, None, {}, - ("z", "p", "est", "se", "tau2", "sigma2"), + ("z", "p", "est", "se", "tau2", "sigma2", "dof"), id="SampleSizeBasedLikelihood_ml", ), pytest.param( @@ -76,7 +76,7 @@ {"method": "reml"}, None, {}, - ("z", "p", "est", "se", "tau2", "sigma2"), + ("z", "p", "est", "se", "tau2", "sigma2", "dof"), id="SampleSizeBasedLikelihood_reml", ), pytest.param( @@ -84,7 +84,7 @@ {"method": "ml"}, None, {}, - ("z", "p", "est", "se", "tau2"), + ("z", "p", "est", "se", "tau2", "dof"), id="VarianceBasedLikelihood_ml", ), pytest.param( @@ -92,7 +92,7 @@ {"method": "reml"}, None, {}, - ("z", "p", "est", "se", "tau2"), + ("z", "p", "est", "se", "tau2", "dof"), id="VarianceBasedLikelihood_reml", ), pytest.param( @@ -100,14 +100,23 @@ {"two_sided": True}, FWECorrector, {"method": "montecarlo", "n_iters": 100, "n_cores": 1}, - ("t", "z"), + ("t", "z", "dof"), id="PermutedOLS", ), ], ) -def test_ibma_smoke(testdata_ibma, meta, meta_kwargs, corrector, corrector_kwargs, maps): +@pytest.mark.parametrize("aggressive_mask", [True, False], ids=["aggressive", "liberal"]) +def test_ibma_smoke( + testdata_ibma, + meta, + aggressive_mask, + meta_kwargs, + corrector, + corrector_kwargs, + maps, +): """Smoke test for IBMA estimators.""" - meta = meta(**meta_kwargs) + meta = meta(aggressive_mask=aggressive_mask, **meta_kwargs) results = meta.fit(testdata_ibma) for expected_map in maps: assert expected_map in results.maps.keys() diff --git a/nimare/tests/test_utils.py b/nimare/tests/test_utils.py index 73ec07f01..706308c93 100644 --- a/nimare/tests/test_utils.py +++ b/nimare/tests/test_utils.py @@ -8,6 +8,7 @@ import pytest from nimare import utils +from nimare.meta.utils import _apply_liberal_mask def test_find_stem(): @@ -183,3 +184,16 @@ def test_mm2vox(): img = utils.get_template(space="mni152_2mm", mask=None) aff = img.affine assert np.array_equal(utils.mm2vox(test, aff), true) + + +def test_apply_liberal_mask(): + """Test _apply_liberal_mask.""" + data = np.array([[1, 2, np.nan, np.nan], [4, np.nan, 6, 5], [0, 8, 9, 3]]) + true_data = [np.array([[1], [4]]), np.array([[2], [8]]), np.array([[6, 5], [9, 3]])] + + pred_data, _, _ = _apply_liberal_mask(data) + + assert len(pred_data) == len(true_data) + + for pred_val, true_val in zip(pred_data, true_data): + assert np.array_equal(pred_val, true_val)