Parallel testing fails when warnings are treated as errors

[IndrajeetPatil]

Uncovered while working on r-lib/lintr#2297.

Minimal reprex

withr::local_tempfile("dir")
withr::local_options(list(usethis.quiet = TRUE))
library(usethis)
pkg_dir <- create_package(dir, open = FALSE, rstudio = FALSE)

withr::with_dir(pkg_dir, {
  use_testthat(3L, parallel = TRUE)
  tests <- c("example-1", "example-2", "example-3", "example-4")
  purrr::walk(tests, use_test, open = FALSE)

  options(warn = 2L)
  testthat::test_dir("tests")
})
#> ✔ | F W  S  OK | Context
#> 
#> ⠋ [ FAIL 0 | WARN 0 | SKIP 0 | PASS 0 ] Starting up...                          
#> ⠙ [ FAIL 0 | WARN 0 | SKIP 0 | PASS 0 ] Starting up...                          
#> ⠹ [ FAIL 0 | WARN 0 | SKIP 0 | PASS 0 ] Starting up...                          
#> ⠸ [ FAIL 0 | WARN 0 | SKIP 0 | PASS 0 ] Starting up...                          
#> ⠼ [ FAIL 0 | WARN 0 | SKIP 0 | PASS 0 ] Starting up...
#> Error in `private$handle_error()`:
#> ! testthat subprocess failed to start, stderr:
#> Error in getNamespace(ns) : attempt to use zero-length variable name
#> Calls: callr_startup_hook ... check_environment -> is_environment -> asNamespace -> getNamespace
#> Caused by error:
#> ! R session crashed with exit code 1
#> Backtrace:
#>      ▆
#>   1. ├─withr::with_dir(...)
#>   2. │ └─base::force(code)
#>   3. └─testthat::test_dir("tests")
#>   4.   └─testthat:::test_files(...)
#>   5.     └─testthat:::test_files_parallel(...)
#>   6.       ├─withr::with_dir(...)
#>   7.       │ └─base::force(code)
#>   8.       ├─testthat::with_reporter(...)
#>   9.       │ └─base::tryCatch(...)
#>  10.       │   └─base (local) tryCatchList(expr, classes, parentenv, handlers)
#>  11.       │     └─base (local) tryCatchOne(expr, names, parentenv, handlers[[1L]])
#>  12.       │       └─base (local) doTryCatch(return(expr), name, parentenv, handler)
#>  13.       └─testthat:::parallel_event_loop_smooth(queue, reporters, ".")
#>  14.         └─queue$poll(poll_time)
#>  15.           └─base::lapply(...)
#>  16.             └─testthat (local) FUN(X[[i]], ...)
#>  17.               └─private$handle_error(msg, i)
#>  18.                 └─rlang::abort(...)

^{Created on 2023-12-01 with reprex v2.0.2}

Standard output and standard error

-- nothing to show --

Session info

sessioninfo::session_info()
#> ─ Session info ───────────────────────────────────────────────────────────────
#>  setting  value
#>  version  R version 4.3.2 (2023-10-31)
#>  os       macOS Sonoma 14.1.1
#>  system   aarch64, darwin20
#>  ui       X11
#>  language (EN)
#>  collate  en_US.UTF-8
#>  ctype    en_US.UTF-8
#>  tz       Europe/Berlin
#>  date     2023-12-01
#>  pandoc   3.1.9 @ /usr/local/bin/ (via rmarkdown)
#>  quarto   1.4.510 @ /usr/local/bin/quarto
#> 
#> ─ Packages ───────────────────────────────────────────────────────────────────
#>  package     * version    date (UTC) lib source
#>  brio          1.1.3      2021-11-30 [1] CRAN (R 4.3.0)
#>  callr         3.7.3      2022-11-02 [1] CRAN (R 4.3.0)
#>  cli           3.6.1.9000 2023-10-03 [1] local
#>  crayon        1.5.2      2022-09-29 [1] CRAN (R 4.3.0)
#>  desc          1.4.2      2022-09-08 [1] CRAN (R 4.3.0)
#>  digest        0.6.33     2023-07-07 [1] CRAN (R 4.3.0)
#>  evaluate      0.23       2023-11-01 [1] CRAN (R 4.3.1)
#>  fansi         1.0.5      2023-10-08 [1] CRAN (R 4.3.1)
#>  fastmap       1.1.1      2023-02-24 [1] CRAN (R 4.3.0)
#>  fs            1.6.3      2023-07-20 [1] CRAN (R 4.3.0)
#>  glue          1.6.2      2022-02-24 [1] CRAN (R 4.3.0)
#>  htmltools     0.5.7      2023-11-03 [1] RSPM (R 4.3.0)
#>  knitr         1.45       2023-10-30 [1] CRAN (R 4.3.1)
#>  lifecycle     1.0.4      2023-11-07 [1] RSPM (R 4.3.0)
#>  magrittr      2.0.3      2022-03-30 [1] CRAN (R 4.3.0)
#>  pillar        1.9.0      2023-03-22 [1] CRAN (R 4.3.0)
#>  pkgconfig     2.0.3      2019-09-22 [1] CRAN (R 4.3.0)
#>  pkgload       1.3.3      2023-09-22 [1] CRAN (R 4.3.1)
#>  processx      3.8.2      2023-06-30 [1] CRAN (R 4.3.0)
#>  ps            1.7.5      2023-04-18 [1] CRAN (R 4.3.0)
#>  purrr         1.0.2      2023-08-10 [1] CRAN (R 4.3.0)
#>  R.cache       0.16.0     2022-07-21 [1] CRAN (R 4.3.0)
#>  R.methodsS3   1.8.2      2022-06-13 [1] CRAN (R 4.3.0)
#>  R.oo          1.25.0     2022-06-12 [1] CRAN (R 4.3.0)
#>  R.utils       2.12.3     2023-11-18 [1] RSPM (R 4.3.0)
#>  R6            2.5.1      2021-08-19 [1] CRAN (R 4.3.0)
#>  reprex        2.0.2      2022-08-17 [1] CRAN (R 4.3.0)
#>  rlang         1.1.2      2023-11-04 [1] RSPM (R 4.3.0)
#>  rmarkdown     2.25       2023-09-18 [1] CRAN (R 4.3.1)
#>  roxygen2      7.2.3      2022-12-08 [1] RSPM (R 4.3.0)
#>  rprojroot     2.0.4      2023-11-05 [1] RSPM (R 4.3.0)
#>  rstudioapi    0.15.0     2023-07-07 [1] CRAN (R 4.3.0)
#>  sessioninfo   1.2.2.9000 2023-11-03 [1] local
#>  stringi       1.8.2      2023-11-23 [1] RSPM (R 4.3.2)
#>  stringr       1.5.1      2023-11-14 [1] RSPM (R 4.3.0)
#>  styler        1.10.2     2023-11-24 [1] Github (r-lib/styler@edf399c)
#>  testthat      3.2.0      2023-10-06 [1] CRAN (R 4.3.1)
#>  tibble        3.2.1      2023-03-20 [1] CRAN (R 4.3.0)
#>  usethis     * 2.2.2      2023-07-06 [1] CRAN (R 4.3.0)
#>  utf8          1.2.4      2023-10-22 [1] CRAN (R 4.3.1)
#>  vctrs         0.6.4      2023-10-12 [1] CRAN (R 4.3.1)
#>  whisker       0.4.1      2022-12-05 [1] CRAN (R 4.3.0)
#>  withr         2.5.2      2023-10-30 [1] CRAN (R 4.3.1)
#>  xfun          0.41       2023-11-01 [1] CRAN (R 4.3.1)
#>  xml2          1.3.5      2023-07-06 [1] CRAN (R 4.3.0)
#>  yaml          2.3.7      2023-01-23 [1] CRAN (R 4.3.0)
#> 
#>  [1] /Library/Frameworks/R.framework/Versions/4.3-arm64/Resources/library
#>  * ── Packages attached to the search path.
#> 
#> ──────────────────────────────────────────────────────────────────────────────

[hadley]

I think my advice would be "don't do that". Is it possible to scope the change to use warn = 2 more narrowly?

[IndrajeetPatil]

Thanks. I reduced the scope to file-level and this works with parallel testing:

pkgload::load_all()
test_script_paths <- testthat::find_test_scripts("tests/testthat")
test_with_warning_as_error <- function(path) {
  withr::local_options(list(warn = 2L))
  testthat::test_file(path, stop_on_failure = TRUE)
}
purrr::walk(test_script_paths, test_with_warning_as_error)