From 7afa2ccc3bddb78cfa501c02663bbbdc9aae9d15 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 30 Nov 2023 15:16:50 +0100 Subject: [PATCH 1/9] Fix issues with glmmPQL --- DESCRIPTION | 2 +- NEWS.md | 3 +++ R/performance_score.R | 16 ++++++++++------ tests/testthat/test-glmmPQL.R | 15 +++++++++++++++ 4 files changed, 29 insertions(+), 7 deletions(-) create mode 100644 tests/testthat/test-glmmPQL.R diff --git a/DESCRIPTION b/DESCRIPTION index ac7741ecd..35a58af93 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Type: Package Package: performance Title: Assessment of Regression Models Performance -Version: 0.10.8.6 +Version: 0.10.8.7 Authors@R: c(person(given = "Daniel", family = "Lüdecke", diff --git a/NEWS.md b/NEWS.md index 5984b7b9d..f3bb5fdd3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -10,6 +10,9 @@ * Fixed issue in `binned_residuals()` for models with binary outcome, where in rare occasions empty bins could occur. +* `performance_score()` should no longer fail for models where scoring rules + can't be calculated. Instead, an informative message is returned. + # performance 0.10.8 ## Changes diff --git a/R/performance_score.R b/R/performance_score.R index eb5ee9b31..cd5b7efd6 100644 --- a/R/performance_score.R +++ b/R/performance_score.R @@ -65,7 +65,7 @@ performance_score <- function(model, verbose = TRUE, ...) { if (minfo$is_ordinal || minfo$is_multinomial) { if (verbose) { - insight::print_color("Can't calculate proper scoring rules for ordinal, multinomial or cumulative link models.\n", "red") + insight::format_alert("Can't calculate proper scoring rules for ordinal, multinomial or cumulative link models.") } return(list(logarithmic = NA, quadratic = NA, spherical = NA)) } @@ -74,10 +74,7 @@ performance_score <- function(model, verbose = TRUE, ...) { if (!is.null(ncol(resp)) && ncol(resp) > 1) { if (verbose) { - insight::print_color( - "Can't calculate proper scoring rules for models without integer response values.\n", - "red" - ) + insight::format_alert("Can't calculate proper scoring rules for models without integer response values.") } return(list(logarithmic = NA, quadratic = NA, spherical = NA)) } @@ -127,7 +124,14 @@ performance_score <- function(model, verbose = TRUE, ...) { } else { datawizard::to_numeric(resp, dummy_factors = FALSE, preserve_levels = TRUE) } - p_y <- prob_fun(resp, mean = pr$pred, pis = pr$pred_zi, sum(resp)) + p_y <- .safe(prob_fun(resp, mean = pr$pred, pis = pr$pred_zi, sum(resp))) + + if (is.null(p_y)) { + if (verbose) { + insight::format_alert("Can't calculate proper scoring rules for this model.") + } + return(list(logarithmic = NA, quadratic = NA, spherical = NA)) + } quadrat_p <- sum(p_y^2) diff --git a/tests/testthat/test-glmmPQL.R b/tests/testthat/test-glmmPQL.R new file mode 100644 index 000000000..64caaccea --- /dev/null +++ b/tests/testthat/test-glmmPQL.R @@ -0,0 +1,15 @@ +skip_if_not_installed("MASS") +test_that("r2", { + example_dat <- data.frame( + prop = c(0.2, 0.2, 0.5, 0.7, 0.1, 1, 1, 1, 0.1), + size = c("small", "small", "small", "large", "large", "large", "large", "small", "small"), + x = c(0.1, 0.1, 0.8, 0.7, 0.6, 0.5, 0.5, 0.1, 0.1), + species = c("sp1", "sp1", "sp2", "sp2", "sp3", "sp3", "sp4", "sp4", "sp4"), + stringsAsFactors = FALSE + ) + mn <- MASS::glmmPQL(prop ~ x + size, + random = ~ 1 | species, + family = "quasibinomial", data = example_dat + ) + expect_message(performance_score(mn), regex = "Cant calculate") +}) From 7778af93b66ebb8f7d302635c7095ea50ee2a9b4 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 30 Nov 2023 16:01:43 +0100 Subject: [PATCH 2/9] remotes --- DESCRIPTION | 1 + 1 file changed, 1 insertion(+) diff --git a/DESCRIPTION b/DESCRIPTION index 35a58af93..b7e41a8f1 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -153,3 +153,4 @@ Config/Needs/website: r-lib/pkgdown, easystats/easystatstemplate Config/rcmdcheck/ignore-inconsequential-notes: true +Remotes: easystats/insight \ No newline at end of file From 306999ce05e9cfdd3b15c3eee770ede64ff142cd Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 30 Nov 2023 17:00:54 +0100 Subject: [PATCH 3/9] final line --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index b7e41a8f1..207139826 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -153,4 +153,4 @@ Config/Needs/website: r-lib/pkgdown, easystats/easystatstemplate Config/rcmdcheck/ignore-inconsequential-notes: true -Remotes: easystats/insight \ No newline at end of file +Remotes: easystats/insight From f9b902f5d1213275437dfce79c0ddd414c105cea Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 1 Dec 2023 08:43:52 +0100 Subject: [PATCH 4/9] fix --- R/performance_score.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/performance_score.R b/R/performance_score.R index cd5b7efd6..e65ae820a 100644 --- a/R/performance_score.R +++ b/R/performance_score.R @@ -124,9 +124,9 @@ performance_score <- function(model, verbose = TRUE, ...) { } else { datawizard::to_numeric(resp, dummy_factors = FALSE, preserve_levels = TRUE) } - p_y <- .safe(prob_fun(resp, mean = pr$pred, pis = pr$pred_zi, sum(resp))) + p_y <- .safe(suppressWarnings(prob_fun(resp, mean = pr$pred, pis = pr$pred_zi, sum(resp)))) - if (is.null(p_y)) { + if (is.null(p_y) || all(is.na(p_y))) { if (verbose) { insight::format_alert("Can't calculate proper scoring rules for this model.") } From cef44fe50a3d5d3e06cc9e55f8a5412ddc2d1846 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 19 Dec 2023 17:38:24 +0100 Subject: [PATCH 5/9] `check_itemscale()` accepts data frames --- DESCRIPTION | 2 +- NAMESPACE | 1 + NEWS.md | 5 ++ R/check_itemscale.R | 78 +++++++++++++++++++++++---- R/performance_score.R | 36 ++++++------- man/check_itemscale.Rd | 20 +++++-- tests/testthat/test-check_itemscale.R | 26 +++++++++ 7 files changed, 135 insertions(+), 33 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 207139826..051302ee6 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Type: Package Package: performance Title: Assessment of Regression Models Performance -Version: 0.10.8.7 +Version: 0.10.8.8 Authors@R: c(person(given = "Daniel", family = "Lüdecke", diff --git a/NAMESPACE b/NAMESPACE index 0bb599f64..970895f14 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -308,6 +308,7 @@ S3method(print,r2_nakagawa_by_group) S3method(print,r2_pseudo) S3method(print,test_likelihoodratio) S3method(print,test_performance) +S3method(print_html,check_itemscale) S3method(print_html,compare_performance) S3method(print_html,test_performance) S3method(print_md,check_itemscale) diff --git a/NEWS.md b/NEWS.md index f3bb5fdd3..bf0e6ac30 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,6 +5,11 @@ * `r2()` for models of class `glmmTMB` without random effects now returns the correct r-squared value for non-mixed models. +* `check_itemscale()` now also accepts data frames as input. In this case, + `factor_index` must be specified, which must be a numeric vector of same + length as number of columns in `x`, where each element is the index of the + factor to which the respective column in `x`. + ## Bug fixes * Fixed issue in `binned_residuals()` for models with binary outcome, where diff --git a/R/check_itemscale.R b/R/check_itemscale.R index 5aa704618..fda758d24 100644 --- a/R/check_itemscale.R +++ b/R/check_itemscale.R @@ -2,11 +2,14 @@ #' @name check_itemscale #' #' @description Compute various measures of internal consistencies -#' applied to (sub)scales, which items were extracted using -#' `parameters::principal_components()`. +#' applied to (sub)scales, which items were extracted using +#' `parameters::principal_components()`. #' #' @param x An object of class `parameters_pca`, as returned by -#' [`parameters::principal_components()`]. +#' [`parameters::principal_components()`], or a data frame. +#' @param factor_index If `x` is a data frame, `factor_index` must be specified. +#' It must be a numeric vector of same length as number of columns in `x`, where +#' each element is the index of the factor to which the respective column in `x`. #' #' @return A list of data frames, with related measures of internal #' consistencies of each subscale. @@ -48,21 +51,53 @@ #' X <- matrix(rnorm(1600), 100, 16) #' Z <- X %*% C #' -#' pca <- principal_components(as.data.frame(Z), rotation = "varimax", n = 3) +#' pca <- parameters::principal_components( +#' as.data.frame(Z), +#' rotation = "varimax", +#' n = 3 +#' ) #' pca #' check_itemscale(pca) +#' +#' # as data frame +#' check_itemscale( +#' as.data.frame(Z), +#' factor_index = parameters::closest_component(pca) +#' ) #' @export -check_itemscale <- function(x) { - if (!inherits(x, "parameters_pca")) { +check_itemscale <- function(x, factor_index = NULL) { + # check for valid input + if (!inherits(x, c("parameters_pca", "data.frame"))) { insight::format_error( - "`x` must be an object of class `parameters_pca`, as returned by `parameters::principal_components()`." + "`x` must be an object of class `parameters_pca`, as returned by `parameters::principal_components()`, or a data frame." # nolint ) } - insight::check_if_installed("parameters") + # if data frame, we need `factor_index` + if (inherits(x, "data.frame") && !inherits(x, "parameters_pca")) { + if (is.null(factor_index)) { + insight::format_error("If `x` is a data frame, `factor_index` must be specified.") + } + if (!is.numeric(factor_index)) { + insight::format_error("`factor_index` must be numeric.") + } + if (length(factor_index) != ncol(x)) { + insight::format_error( + "`factor_index` must be of same length as number of columns in `x`.", + "Each element of `factor_index` must be the index of the factor to which the respective column in `x` belongs to." # nolint + ) + } + } - dataset <- attributes(x)$dataset - subscales <- parameters::closest_component(x) + # assign data and factor index + if (inherits(x, "parameters_pca")) { + insight::check_if_installed("parameters") + dataset <- attributes(x)$dataset + subscales <- parameters::closest_component(x) + } else { + dataset <- x + subscales <- factor_index + } out <- lapply(sort(unique(subscales)), function(.subscale) { columns <- names(subscales)[subscales == .subscale] @@ -123,3 +158,26 @@ print.check_itemscale <- function(x, digits = 2, ...) { zap_small = TRUE )) } + + +#' @export +print_html.check_itemscale <- function(x, digits = 2, ...) { + x <- lapply(seq_along(x), function(i) { + out <- x[[i]] + attr(out, "table_caption") <- sprintf( + "Component %i: Mean inter-item-correlation = %.3f, Cronbach's alpha = %.3f", + i, + attributes(out)$item_intercorrelation, + attributes(out)$cronbachs_alpha + ) + out + }) + insight::export_table( + x, + caption = "Description of (Sub-)Scales", + digits = digits, + format = "html", + missing = "", + zap_small = TRUE + ) +} diff --git a/R/performance_score.R b/R/performance_score.R index e65ae820a..5ca3fdc84 100644 --- a/R/performance_score.R +++ b/R/performance_score.R @@ -209,27 +209,25 @@ print.performance_score <- function(x, ...) { pred_zi <- NULL tryCatch( - { - if (inherits(model, "MixMod")) { - pred <- stats::predict(model, type = "subject_specific") - pred_zi <- if (!is.null(model$gammas)) attr(pred, "zi_probs") - } else if (inherits(model, "glmmTMB")) { - pred <- stats::predict(model, type = "response") - pred_zi <- stats::predict(model, type = "zprob") - } else if (inherits(model, c("hurdle", "zeroinfl"))) { - pred <- stats::predict(model, type = "response") - pred_zi <- stats::predict(model, type = "zero") - } else if (inherits(model, c("clm", "clm2", "clmm"))) { - pred <- stats::predict(model) - } else if (all(inherits(model, c("stanreg", "lmerMod"), which = TRUE)) > 0) { - insight::check_if_installed("rstanarm") - pred <- colMeans(rstanarm::posterior_predict(model)) - } else { - pred <- stats::predict(model, type = "response") - } + if (inherits(model, "MixMod")) { + pred <- stats::predict(model, type = "subject_specific") + pred_zi <- if (!is.null(model$gammas)) attr(pred, "zi_probs") + } else if (inherits(model, "glmmTMB")) { + pred <- stats::predict(model, type = "response") + pred_zi <- stats::predict(model, type = "zprob") + } else if (inherits(model, c("hurdle", "zeroinfl"))) { + pred <- stats::predict(model, type = "response") + pred_zi <- stats::predict(model, type = "zero") + } else if (inherits(model, c("clm", "clm2", "clmm"))) { + pred <- stats::predict(model) + } else if (all(inherits(model, c("stanreg", "lmerMod"), which = TRUE)) > 0) { + insight::check_if_installed("rstanarm") + pred <- colMeans(rstanarm::posterior_predict(model)) + } else { + pred <- stats::predict(model, type = "response") }, error = function(e) { - return(NULL) + NULL } ) diff --git a/man/check_itemscale.Rd b/man/check_itemscale.Rd index 7f790b1d2..a5ada3875 100644 --- a/man/check_itemscale.Rd +++ b/man/check_itemscale.Rd @@ -4,11 +4,15 @@ \alias{check_itemscale} \title{Describe Properties of Item Scales} \usage{ -check_itemscale(x) +check_itemscale(x, factor_index = NULL) } \arguments{ \item{x}{An object of class \code{parameters_pca}, as returned by -\code{\link[parameters:principal_components]{parameters::principal_components()}}.} +\code{\link[parameters:principal_components]{parameters::principal_components()}}, or a data frame.} + +\item{factor_index}{If \code{x} is a data frame, \code{factor_index} must be specified. +It must be a numeric vector of same length as number of columns in \code{x}, where +each element is the index of the factor to which the respective column in \code{x}.} } \value{ A list of data frames, with related measures of internal @@ -51,9 +55,19 @@ set.seed(17) X <- matrix(rnorm(1600), 100, 16) Z <- X \%*\% C -pca <- principal_components(as.data.frame(Z), rotation = "varimax", n = 3) +pca <- parameters::principal_components( + as.data.frame(Z), + rotation = "varimax", + n = 3 +) pca check_itemscale(pca) + +# as data frame +check_itemscale( + as.data.frame(Z), + factor_index = parameters::closest_component(pca) +) \dontshow{\}) # examplesIf} } \references{ diff --git a/tests/testthat/test-check_itemscale.R b/tests/testthat/test-check_itemscale.R index 119283d37..d64dbaea6 100644 --- a/tests/testthat/test-check_itemscale.R +++ b/tests/testthat/test-check_itemscale.R @@ -25,4 +25,30 @@ test_that("check_convergence", { tolerance = 1e-4, ignore_attr = TRUE ) + comp <- parameters::closest_component(pca) + out2 <- check_itemscale(d, comp) + expect_equal( + out[[1]]$Mean, + out2[[1]]$Mean, + tolerance = 1e-4, + ignore_attr = TRUE + ) + expect_equal( + out[[1]]$Difficulty, + out2[[1]]$Difficulty, + tolerance = 1e-4, + ignore_attr = TRUE + ) + expect_error( + check_itemscale(d), + regex = "If `x` is a data" + ) + expect_error( + check_itemscale(d, factor_index = 1:8), + regex = "`factor_index` must be of same" + ) + expect_error( + check_itemscale(d, factor_index = factor(comp)), + regex = "`factor_index` must be numeric." + ) }) From b8c99adb5562a04754c4fe022eee92b65d07ed73 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 19 Dec 2023 17:38:51 +0100 Subject: [PATCH 6/9] news --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index bf0e6ac30..6fe789f56 100644 --- a/NEWS.md +++ b/NEWS.md @@ -10,6 +10,8 @@ length as number of columns in `x`, where each element is the index of the factor to which the respective column in `x`. +* `check_itemscale()` gets a `print_html()` method. + ## Bug fixes * Fixed issue in `binned_residuals()` for models with binary outcome, where From a3b5ba4d43f2fe0c3ecf279bf8b8243ff033c03e Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 19 Dec 2023 17:42:07 +0100 Subject: [PATCH 7/9] add test --- tests/testthat/_snaps/check_itemscale.md | 24 ++++++++++++++++++++++++ tests/testthat/test-check_itemscale.R | 3 ++- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 tests/testthat/_snaps/check_itemscale.md diff --git a/tests/testthat/_snaps/check_itemscale.md b/tests/testthat/_snaps/check_itemscale.md new file mode 100644 index 000000000..48e0bd482 --- /dev/null +++ b/tests/testthat/_snaps/check_itemscale.md @@ -0,0 +1,24 @@ +# check_itemscale + + Code + print(out) + Output + # Description of (Sub-)ScalesComponent 1 + + Item | Missings | Mean | SD | Skewness | Difficulty | Discrimination | alpha if deleted + ----------------------------------------------------------------------------------------- + b | 0 | 5.02 | 0.79 | -0.04 | 0.84 | 0.06 | -0.55 + e | 0 | 2.12 | 0.81 | -0.22 | 0.35 | -0.09 | -0.03 + f | 0 | 2.00 | 0.82 | 0.00 | 0.33 | -0.16 | 0.17 + + Mean inter-item-correlation = -0.046 Cronbach's alpha = -0.159 + Component 2 + + Item | Missings | Mean | SD | Skewness | Difficulty | Discrimination | alpha if deleted + ----------------------------------------------------------------------------------------- + a | 0 | 5.02 | 0.83 | -0.04 | 0.84 | 0.21 | -0.18 + c | 0 | 4.74 | 0.81 | 0.51 | 0.79 | -0.04 | 0.41 + d | 0 | 2.07 | 0.79 | -0.13 | 0.34 | 0.13 | 0.04 + + Mean inter-item-correlation = 0.067 Cronbach's alpha = 0.178 + diff --git a/tests/testthat/test-check_itemscale.R b/tests/testthat/test-check_itemscale.R index d64dbaea6..0c2d605fc 100644 --- a/tests/testthat/test-check_itemscale.R +++ b/tests/testthat/test-check_itemscale.R @@ -1,4 +1,4 @@ -test_that("check_convergence", { +test_that("check_itemscale", { skip_if_not_installed("parameters") set.seed(123) @@ -25,6 +25,7 @@ test_that("check_convergence", { tolerance = 1e-4, ignore_attr = TRUE ) + expect_snapshot(print(out)) comp <- parameters::closest_component(pca) out2 <- check_itemscale(d, comp) expect_equal( From d3babbfffc1f34e39a3d2914df2d0e40263faec1 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 19 Dec 2023 17:42:47 +0100 Subject: [PATCH 8/9] on win only --- tests/testthat/_snaps/{ => windows}/check_itemscale.md | 0 tests/testthat/test-check_itemscale.R | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/testthat/_snaps/{ => windows}/check_itemscale.md (100%) diff --git a/tests/testthat/_snaps/check_itemscale.md b/tests/testthat/_snaps/windows/check_itemscale.md similarity index 100% rename from tests/testthat/_snaps/check_itemscale.md rename to tests/testthat/_snaps/windows/check_itemscale.md diff --git a/tests/testthat/test-check_itemscale.R b/tests/testthat/test-check_itemscale.R index 0c2d605fc..95ab4e169 100644 --- a/tests/testthat/test-check_itemscale.R +++ b/tests/testthat/test-check_itemscale.R @@ -25,7 +25,7 @@ test_that("check_itemscale", { tolerance = 1e-4, ignore_attr = TRUE ) - expect_snapshot(print(out)) + expect_snapshot(print(out), variant = "windows") comp <- parameters::closest_component(pca) out2 <- check_itemscale(d, comp) expect_equal( From 98d3ed9b52edcd217b7fcf4e7801d1a7979a165e Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 19 Dec 2023 18:10:32 +0100 Subject: [PATCH 9/9] typo --- tests/testthat/test-glmmPQL.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test-glmmPQL.R b/tests/testthat/test-glmmPQL.R index 64caaccea..6c1713cc8 100644 --- a/tests/testthat/test-glmmPQL.R +++ b/tests/testthat/test-glmmPQL.R @@ -11,5 +11,5 @@ test_that("r2", { random = ~ 1 | species, family = "quasibinomial", data = example_dat ) - expect_message(performance_score(mn), regex = "Cant calculate") + expect_message(performance_score(mn), regex = "Can't calculate") })