diff --git a/DHARMa/R/tests.R b/DHARMa/R/tests.R index 2be033ea..e1027bef 100644 --- a/DHARMa/R/tests.R +++ b/DHARMa/R/tests.R @@ -153,6 +153,7 @@ testQuantiles <- function(simulationOutput, predictor = NULL, quantiles = c(0.25 #' @param type either default, bootstrap or binomial. See details #' @param nBoot number of boostrap replicates. Only used ot type = "bootstrap" #' @param plot if TRUE, the function will create an additional plot +#' @param plotBoostrap if plot should be produced of outlier frequencies calculated under the bootstrap #' @details DHARMa residuals are created by simulating from the fitted model, and comparing the simulated values to the observed data. It can occur that all simulated values are higher or smaller than the observed data, in which case they get the residual value of 0 and 1, respectively. I refer to these values as simulation outliers, or simply outliers. #' #' Because no data was simulated in the range of the observed value, we don't know "how strongly" these values deviate from the model expectation, so the term "outlier" should be used with a grain of salt. It is not a judgment about the magnitude of the residual deviation, but simply a dichotomous sign that we are outside the simulated range. Moreover, the number of outliers will decrease as we increase the number of simulations. @@ -169,7 +170,7 @@ testQuantiles <- function(simulationOutput, predictor = NULL, quantiles = c(0.25 #' @author Florian Hartig #' @seealso [testResiduals], [testUniformity], [testOutliers], [testDispersion], [testZeroInflation], [testGeneric], [testTemporalAutocorrelation], [testSpatialAutocorrelation], [testQuantiles], [testCategorical] #' @export -testOutliers <- function(simulationOutput, alternative = c("two.sided", "greater", "less"), margin = c("both", "upper", "lower"), type = c("default","bootstrap", "binomial"), nBoot = 100, plot = TRUE){ +testOutliers <- function(simulationOutput, alternative = c("two.sided", "greater", "less"), margin = c("both", "upper", "lower"), type = c("default","bootstrap", "binomial"), nBoot = 100, plot = TRUE, plotBoostrap = FALSE){ # check inputs alternative = match.arg(alternative) @@ -284,12 +285,17 @@ testOutliers <- function(simulationOutput, alternative = c("two.sided", "greater names(out$parameter) = "observations" out$estimate = outliers names(out$estimate) = paste("outlier frequency (expected:", mean(frequBoot),")") + + if(plotBoostrap == T){ + hist(frequBoot, xlim = range(frequBoot, outliers), col = "lightgrey", main = "Bootstrapped outlier frequency") + abline(v = mean(frequBoot), col = 1, lwd = 2) + abline(v = outliers, col = "red", lwd = 2) + + # legend("center", c(paste("p=", round(out$p.value, digits = 5)), paste("Deviation ", ifelse(out$p.value < 0.05, "significant", "n.s."))), text.col = ifelse(out$p.value < 0.05, "red", "black" )) + } if(plot == T) { - - opar <- par(mfrow = c(1,2)) - on.exit(par(opar)) - + hist(simulationOutput, main = "") main = ifelse(out$p.value <= 0.05, @@ -299,12 +305,6 @@ testOutliers <- function(simulationOutput, alternative = c("two.sided", "greater title(main = main, cex.main = 1, col.main = ifelse(out$p.value <= 0.05, "red", "black")) - hist(frequBoot, xlim = range(frequBoot, outliers), col = "lightgrey") - abline(v = mean(frequBoot), col = 1, lwd = 2) - abline(v = outliers, col = "red", lwd = 2) - - # legend("center", c(paste("p=", round(out$p.value, digits = 5)), paste("Deviation ", ifelse(out$p.value < 0.05, "significant", "n.s."))), text.col = ifelse(out$p.value < 0.05, "red", "black" )) - } } return(out) diff --git a/DHARMa/man/testCategorical.Rd b/DHARMa/man/testCategorical.Rd index 9d3a0f8e..26f855cd 100644 --- a/DHARMa/man/testCategorical.Rd +++ b/DHARMa/man/testCategorical.Rd @@ -31,12 +31,12 @@ testData = createData(sampleSize = 100, overdispersion = 0.5, randomEffectVarian fittedModel <- glm(observedResponse ~ Environment1 , family = "poisson", data = testData) simulationOutput <- simulateResiduals(fittedModel = fittedModel) -# the plot function runs 4 tests +# the plot function shows 2 plots and runs 4 tests # i) KS test i) Dispersion test iii) Outlier test iv) quantile test plot(simulationOutput, quantreg = TRUE) # testResiduals tests distribution, dispersion and outliers -# testResiduals(simulationOutput) +testResiduals(simulationOutput) ####### Individual tests ####### diff --git a/DHARMa/man/testGeneric.Rd b/DHARMa/man/testGeneric.Rd index 5c2ef39a..f99216e3 100644 --- a/DHARMa/man/testGeneric.Rd +++ b/DHARMa/man/testGeneric.Rd @@ -37,12 +37,12 @@ testData = createData(sampleSize = 100, overdispersion = 0.5, randomEffectVarian fittedModel <- glm(observedResponse ~ Environment1 , family = "poisson", data = testData) simulationOutput <- simulateResiduals(fittedModel = fittedModel) -# the plot function runs 4 tests +# the plot function shows 2 plots and runs 4 tests # i) KS test i) Dispersion test iii) Outlier test iv) quantile test plot(simulationOutput, quantreg = TRUE) # testResiduals tests distribution, dispersion and outliers -# testResiduals(simulationOutput) +testResiduals(simulationOutput) ####### Individual tests ####### diff --git a/DHARMa/man/testOutliers.Rd b/DHARMa/man/testOutliers.Rd index 7b306a51..f3762f4f 100644 --- a/DHARMa/man/testOutliers.Rd +++ b/DHARMa/man/testOutliers.Rd @@ -6,7 +6,8 @@ \usage{ testOutliers(simulationOutput, alternative = c("two.sided", "greater", "less"), margin = c("both", "upper", "lower"), type = c("default", - "bootstrap", "binomial"), nBoot = 100, plot = TRUE) + "bootstrap", "binomial"), nBoot = 100, plot = TRUE, + plotBoostrap = FALSE) } \arguments{ \item{simulationOutput}{an object of class DHARMa, either created via \link{simulateResiduals} for supported models or by \link{createDHARMa} for simulations created outside DHARMa, or a supported model. Providing a supported model directly is discouraged, because simulation settings cannot be changed in this case.} @@ -20,6 +21,8 @@ testOutliers(simulationOutput, alternative = c("two.sided", "greater", \item{nBoot}{number of boostrap replicates. Only used ot type = "bootstrap"} \item{plot}{if TRUE, the function will create an additional plot} + +\item{plotBoostrap}{if plot should be produced of outlier frequencies calculated under the bootstrap} } \description{ This function tests if the number of observations outside the simulatio envelope are larger or smaller than expected @@ -31,7 +34,7 @@ Because no data was simulated in the range of the observed value, we don't know To test if the outliers are a concern, testOutliers implements 2 options (bootstrap, binomial), which can be chosen via the parameter "type". The third option (default) chooses bootstrap for integer-valued distribubtions with nObs < 500, and else binomial. -The binomial test considers that under the null hypothesis that the model is correct, and for continuous distributions (i.e. data and the model distribution are identical and continous), the probability that a given observation is higher than all simulations is 1/(nSim +1), and binomial distributed. The testOutlier function can test this null hypothesis via type = "binomial". In principle, it would be nice if we could extend this idea to integer-valued distributions, which are randomized via the PIT procedure (see \link{simulateResiduals}), the rate of "true" outliers is more difficult to calculate, and in general not 1/(nSim +1). The testOutlier function implements a small tweak that calculates the rate of residuals that are closer than 1/(nSim+1) to the 0/1 border, which roughly occur at a rate of nData /(nSim +1). This approximate value, however, is generally not exact, and may be particularly off non-bounded integer-valued distributions (such as Poisson or neg binom). +The binomial test considers that under the null hypothesis that the model is correct, and for continuous distributions (i.e. data and the model distribution are identical and continous), the probability that a given observation is higher than all simulations is 1/(nSim +1), and binomial distributed. The testOutlier function can test this null hypothesis via type = "binomial". In principle, it would be nice if we could extend this idea to integer-valued distributions, which are randomized via the PIT procedure (see \link{simulateResiduals}), the rate of "true" outliers is more difficult to calculate, and in general not 1/(nSim +1). The testOutlier function implements a small tweak that calculates the rate of residuals that are closer than 1/(nSim+1) to the 0/1 border, which roughly occur at a rate of nData /(nSim +1). This approximate value, however, is generally not exact, and may be particularly off non-bounded integer-valued distributions (such as Poisson or Negative Binomial). For this reason, the testOutlier function implements an alternative procedure that uses the bootstrap to generate a simulation-based expectation for the outliers. It is recommended to use the bootstrap for integer-valued distributions (and integer-valued only, because it has no advantage for continuous distributions, ideally with reasonably high values of nSim and nBoot (I recommend at least 1000 for both). Because of the high runtime, however, this option is switched off for type = default when nObs > 500.