From 25771eb01107a2cab9efc64a4957dcf81b1435e6 Mon Sep 17 00:00:00 2001 From: Eric Scott Date: Wed, 2 Oct 2024 12:36:11 -0700 Subject: [PATCH 01/10] demonstrate with tar_terra_rast() --- DESCRIPTION | 2 +- R/tar-terra-rast.R | 24 +++++------------------- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 15df6d5..63a6b20 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -32,7 +32,7 @@ Language: en-GB Roxygen: list(markdown = TRUE) RoxygenNote: 7.3.2 Imports: - targets (>= 1.7.0), + targets (>= 1.8.0), rlang (>= 1.1.3), cli (>= 3.6.2) Suggests: diff --git a/R/tar-terra-rast.R b/R/tar-terra-rast.R index f4d11b4..1caab6e 100644 --- a/R/tar-terra-rast.R +++ b/R/tar-terra-rast.R @@ -54,7 +54,6 @@ tar_terra_rast <- function(name, cue = targets::tar_option_get("cue"), description = targets::tar_option_get("description")) { filetype <- filetype %||% "GTiff" - gdal <- gdal %||% character(0) #check that filetype option is available drv <- get_gdal_available_driver_list("raster") @@ -95,16 +94,14 @@ tar_terra_rast <- function(name, terra::writeRaster( object, path, - filetype = Sys.getenv("GEOTARGETS_GDAL_RASTER_DRIVER"), + filetype = filetype, overwrite = TRUE, - gdal = strsplit( - Sys.getenv("GEOTARGETS_GDAL_RASTER_CREATION_OPTIONS", - unset = ";"), - ";")[[1]] + gdal = gdal ) }, marshal = function(object) terra::wrap(object), - unmarshal = function(object) terra::unwrap(object) + unmarshal = function(object) terra::unwrap(object), + substitute = list(filetype = filetype, gdal = gdal) ), repository = repository, iteration = "list", #only "list" works right now @@ -113,18 +110,7 @@ tar_terra_rast <- function(name, garbage_collection = garbage_collection, deployment = deployment, priority = priority, - resources = utils::modifyList( - targets::tar_resources( - custom_format = targets::tar_resources_custom_format( - #these envvars are used in write function of format - envvars = c( - "GEOTARGETS_GDAL_RASTER_DRIVER" = filetype, - "GEOTARGETS_GDAL_RASTER_CREATION_OPTIONS" = ( - paste0(gdal, collapse = ";") - ) - ) - ) - ), resources), + resources = resources, storage = storage, retrieval = retrieval, cue = cue, From 8ed08c531dd19e18126aeb0c14b2bd81672269cc Mon Sep 17 00:00:00 2001 From: Eric Scott Date: Wed, 9 Oct 2024 09:41:26 -0700 Subject: [PATCH 02/10] test that changing filetype invalidates target --- tests/testthat/test-tar-terra.R | 35 +++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/testthat/test-tar-terra.R b/tests/testthat/test-tar-terra.R index 9312e46..f31f956 100644 --- a/tests/testthat/test-tar-terra.R +++ b/tests/testthat/test-tar-terra.R @@ -166,3 +166,38 @@ targets::tar_test("user resources are passed correctly", { ) }) +tar_test("That changing filetype invalidates a target", { + targets::tar_script({ + library(targets) + library(geotargets) + library(terra) + + list( + tar_terra_rast( + r, + rast(system.file("ex/elev.tif", package="terra")), + filetype = "COG" + ) + ) + }) + tar_make() + first <- tar_meta()$time[2] + targets::tar_script({ + library(targets) + library(geotargets) + library(terra) + + list( + tar_terra_rast( + r, + rast(system.file("ex/elev.tif", package="terra")), + filetype = "GTiff" + ) + ) + }) + tar_make() + second <- tar_meta()$time[2] + #There's gotta be a better way to test if a target is invalidated + expect_gt(second, first) + +}) From cd32d482e5ad4e543b4cde65ca5adc4173e51422 Mon Sep 17 00:00:00 2001 From: Eric Scott Date: Wed, 9 Oct 2024 15:57:35 -0700 Subject: [PATCH 03/10] use new substitute arg to tar_format() --- R/tar-terra-rast.R | 3 +- R/tar-terra-vect.R | 99 +++++++++++++--------------------------------- 2 files changed, 29 insertions(+), 73 deletions(-) diff --git a/R/tar-terra-rast.R b/R/tar-terra-rast.R index 1caab6e..337c1b8 100644 --- a/R/tar-terra-rast.R +++ b/R/tar-terra-rast.R @@ -53,13 +53,14 @@ tar_terra_rast <- function(name, retrieval = targets::tar_option_get("retrieval"), cue = targets::tar_option_get("cue"), description = targets::tar_option_get("description")) { + check_pkg_installed("terra") + filetype <- filetype %||% "GTiff" #check that filetype option is available drv <- get_gdal_available_driver_list("raster") filetype <- rlang::arg_match0(filetype, drv$name) - check_pkg_installed("terra") #ensure that user-passed `resources` doesn't include `custom_format` if ("custom_format" %in% names(resources)) { diff --git a/R/tar-terra-vect.R b/R/tar-terra-vect.R index f98e002..8a102af 100644 --- a/R/tar-terra-vect.R +++ b/R/tar-terra-vect.R @@ -62,6 +62,8 @@ tar_terra_vect <- function(name, retrieval = targets::tar_option_get("retrieval"), cue = targets::tar_option_get("cue"), description = targets::tar_option_get("description")) { + check_pkg_installed("terra") + filetype <- filetype %||% "GeoJSON" gdal <- gdal %||% "ENCODING=UTF-8" @@ -69,7 +71,6 @@ tar_terra_vect <- function(name, drv <- get_gdal_available_driver_list("vector") filetype <- rlang::arg_match0(filetype, drv$name) - check_pkg_installed("terra") #ensure that user-passed `resources` doesn't include `custom_format` if ("custom_format" %in% names(resources)) { @@ -91,20 +92,36 @@ tar_terra_vect <- function(name, tidy_eval = tidy_eval ) - format <- ifelse( - test = filetype == "ESRI Shapefile", - #special handling of ESRI shapefiles because the output is a dir of multiple files. - yes = create_format_terra_vect_shz(), - no = create_format_terra_vect() - ) - targets::tar_target_raw( name = name, command = command, pattern = pattern, packages = packages, library = library, - format = format, + format = targets::tar_format( + read = function(path) { + if (filetype == "ESRI Shapefile") { + terra::vect(paste0("/vsizip/{", path, "}")) + } else { + terra::vect(path) + } + }, + write = function(object, path) { + terra::writeVector( + object, + filename = ifelse(filetype == "ESRI Shapefile", paste0(path, ".shz"), path), + filetype = filetype, + overwrite = TRUE, + options = gdal + ) + if (filetype == "ESRI Shapefile") { + file.rename(paste0(path, ".shz"), path) + } + }, + marshal = function(object) terra::wrap(object), + unmarshal = function(object) terra::unwrap(object), + substitute = list(filetype = filetype, gdal = gdal) + ), repository = repository, iteration = "list", #only "list" works for now error = error, @@ -112,72 +129,10 @@ tar_terra_vect <- function(name, garbage_collection = garbage_collection, deployment = deployment, priority = priority, - resources = utils::modifyList( - targets::tar_resources( - custom_format = targets::tar_resources_custom_format( - #these envvars are used in write function of format - envvars = c( - "GEOTARGETS_GDAL_VECTOR_DRIVER" = filetype, - "GEOTARGETS_GDAL_VECTOR_CREATION_OPTIONS" = ( - paste0(gdal, collapse = ";") - ) - ) - ) - ), resources), + resources = resources, storage = storage, retrieval = retrieval, cue = cue, description = description ) } - - -#' @noRd -create_format_terra_vect <- function() { - - check_pkg_installed("terra") - - targets::tar_format( - read = function(path) terra::vect(path), - write = function(object, path) { - terra::writeVector( - object, - path, - filetype = Sys.getenv("GEOTARGETS_GDAL_VECTOR_DRIVER"), - overwrite = TRUE, - options = strsplit( - Sys.getenv("GEOTARGETS_GDAL_VECTOR_CREATION_OPTIONS", - unset = ";"), - ";")[[1]] - ) - }, - marshal = function(object) terra::wrap(object), - unmarshal = function(object) terra::unwrap(object) - ) -} - -#' Special handling for ESRI Shapefiles -#' @noRd -create_format_terra_vect_shz <- function() { - - check_pkg_installed("terra") - - targets::tar_format( - read = function(path) terra::vect(paste0("/vsizip/{", path, "}")), - write = function(object, path) { - terra::writeVector( - x = object, - filename = paste0(path, ".shz"), - filetype = "ESRI Shapefile", - overwrite = TRUE, - options = strsplit( - Sys.getenv("GEOTARGETS_GDAL_VECTOR_CREATION_OPTIONS", - unset = ";"), - ";")[[1]] - ) - file.rename(paste0(path, ".shz"), path) - }, - marshal = function(object) terra::wrap(object), - unmarshal = function(object) terra::unwrap(object) - ) -} From 233e4300052829f3771690b10cc48025c213389d Mon Sep 17 00:00:00 2001 From: Eric Scott Date: Wed, 9 Oct 2024 16:23:22 -0700 Subject: [PATCH 04/10] use substitute arg instead of env vars in resources --- R/tar-stars.R | 98 +++++++++++++++++++-------------------------------- 1 file changed, 37 insertions(+), 61 deletions(-) diff --git a/R/tar-stars.R b/R/tar-stars.R index acb6564..6dee703 100644 --- a/R/tar-stars.R +++ b/R/tar-stars.R @@ -204,52 +204,6 @@ tar_stars_raw <- function(name, driver <- rlang::arg_match0(driver, drv$name) - .read_stars <- eval(substitute( - function(path) { - - ## TODO: it appears envvar custom resources do not work in read function? - READ_FUN <- stars::read_stars - # mdim <- as.logical(Sys.getenv("GEOTARGETS_GDAL_RASTER_MDIM", unset = FALSE)) - if (mdim) { - READ_FUN <- stars::read_mdim - } - - # ncdf <- as.logical(Sys.getenv("GEOTARGETS_USE_NCMETA", unset = FALSE)) - if (ncdf && requireNamespace("ncmeta")) { - READ_FUN <- stars::read_ncdf - } - - # proxy <- as.logical(Sys.getenv("GEOTARGETS_PROXY", unset = FALSE)) - READ_FUN(path, proxy = proxy) - - }, list(ncdf = ncdf, mdim = mdim, proxy = proxy))) - - # TODO: should multidimensional array use the same `options` as 2D? - .write_stars <- function(object, path) { - - WRITE_FUN <- stars::write_stars - - mdim <- as.logical(Sys.getenv("GEOTARGETS_GDAL_RASTER_MDIM", - unset = FALSE)) - if (mdim) { - WRITE_FUN <- stars::write_mdim - } - - dr <- Sys.getenv("GEOTARGETS_GDAL_RASTER_DRIVER") - - # stars requires character(0), not "", for no options set - co <- Sys.getenv("GEOTARGETS_GDAL_RASTER_CREATION_OPTIONS") - co2 <- strsplit(co, ";")[[1]] - - WRITE_FUN( - object, - path, - overwrite = TRUE, - driver = dr, - options = co - ) - } - targets::tar_target_raw( name = name, command = command, @@ -257,10 +211,42 @@ tar_stars_raw <- function(name, packages = packages, library = library, format = targets::tar_format( - read = .read_stars, - write = .write_stars, - marshal = function(object) object, # Not currently used - unmarshal = function(object) object + read = function(path) { + if (ncdf && requireNamespace("ncmeta")) { + stars::read_ncdf(path, proxy = proxy) + } else if (isTRUE(mdim)) { + stars::read_mdim(path, proxy = proxy) + } else { + stars::read_stars(path, proxy = proxy) + } + }, + write = function(object, path) { + if (mdim) { + stars::write_mdim( + object, + path, + overwrite = TRUE, + driver = driver, + options = options + ) + } else { + stars::write_stars( + object, + path, + overwrite = TRUE, + driver = driver, + options = options + ) + } + + }, + substitute = list( + ncdf = ncdf, + mdim = mdim, + proxy = proxy, + driver = driver, + options = options + ) ), repository = repository, iteration = "list", #the only option that works @@ -269,17 +255,7 @@ tar_stars_raw <- function(name, garbage_collection = garbage_collection, deployment = deployment, priority = priority, - resources = utils::modifyList(targets::tar_resources( - custom_format = targets::tar_resources_custom_format( - #these envvars are used in read and write functions of format - envvars = c("GEOTARGETS_GDAL_RASTER_DRIVER" = driver, - "GEOTARGETS_GDAL_RASTER_CREATION_OPTIONS" = - paste0(options, collapse = ";"), - "GEOTARGETS_GDAL_RASTER_MDIM" = mdim, - "GEOTARGETS_PROXY" = proxy, - "GEOTARGETS_USE_NCMETA" = ncdf) - ) - ), resources), + resources = resources, storage = storage, retrieval = retrieval, cue = cue, From fb4ce4ec856396b13ac438258a7bbf68254bb60f Mon Sep 17 00:00:00 2001 From: Eric Scott Date: Wed, 9 Oct 2024 17:03:01 -0700 Subject: [PATCH 05/10] update collections format to use substitute arg --- R/tar-terra-sprc.R | 120 +++++++++++++++++++++++++-------------------- 1 file changed, 67 insertions(+), 53 deletions(-) diff --git a/R/tar-terra-sprc.R b/R/tar-terra-sprc.R index dc03472..576e818 100644 --- a/R/tar-terra-sprc.R +++ b/R/tar-terra-sprc.R @@ -99,11 +99,11 @@ tar_terra_sprc <- function(name, name = name, command = command, pattern = pattern, + type = "sprc", filetype = filetype, gdal = gdal, packages = packages, library = library, - format = format_terra_collections(type = "sprc"), repository = repository, error = error, memory = memory, @@ -216,12 +216,12 @@ tar_terra_sds <- function(name, tar_terra_collection_raw( name = name, command = command, - pattern = pattern, + type = "sds", filetype = filetype, + pattern = pattern, gdal = gdal, packages = packages, library = library, - format = format_terra_collections(type = "sds"), repository = repository, error = error, memory = memory, @@ -243,12 +243,12 @@ tar_terra_sds <- function(name, tar_terra_collection_raw <- function( name, command, + type = type, filetype = geotargets_option_get("gdal.raster.driver"), gdal = geotargets_option_get("gdal.raster.creation.options"), pattern = NULL, packages = targets::tar_option_get("packages"), library = targets::tar_option_get("library"), - format = format_terra_collections(), repository = targets::tar_option_get("repository"), error = targets::tar_option_get("error"), memory = targets::tar_option_get("memory"), @@ -261,13 +261,38 @@ tar_terra_collection_raw <- function( cue = targets::tar_option_get("cue"), description = targets::tar_option_get("description") ) { + targets::tar_target_raw( name = name, command = command, pattern = pattern, packages = packages, library = library, - format = format, + format = targets::tar_format( + read = switch(type, + "sprc" = function(path) terra::sprc(path), + "sds" = function(path) terra::sds(path) + ), + write = function(object, path) { + for (i in seq(object)) { + if (i > 1) { + opt <- "APPEND_SUBDATASET=YES" + } else { + opt <- "" + } + terra::writeRaster( + x = object[i], + filename = path, + filetype = filetype, + overwrite = (i == 1), + gdal = c(opt, gdal) + ) + } + }, + marshal = function(object) terra::wrap(object), + unmarshal = function(object) terra::unwrap(object), + substitute = list(type = type, gdal = gdal, filetype = filetype) + ), repository = repository, iteration = "list", error = error, @@ -275,18 +300,7 @@ tar_terra_collection_raw <- function( garbage_collection = garbage_collection, deployment = deployment, priority = priority, - resources = utils::modifyList( - targets::tar_resources( - custom_format = targets::tar_resources_custom_format( - #these envvars are used in write function of format - envvars = c( - "GEOTARGETS_GDAL_RASTER_DRIVER" = filetype, - "GEOTARGETS_GDAL_RASTER_CREATION_OPTIONS" = ( - paste0(gdal, collapse = ";") - ) - ) - ) - ), resources), + resources = resources, storage = storage, retrieval = retrieval, cue = cue, @@ -295,39 +309,39 @@ tar_terra_collection_raw <- function( } - - -#' Format function for sprc and sds -#' @noRd -format_terra_collections <- function(type = c("sprc", "sds")) { - type <- match.arg(type) - targets::tar_format( - read = switch(type, - "sprc" = function(path) terra::sprc(path), - "sds" = function(path) terra::sds(path) - ), - write = function(object, path) { - gdal <- strsplit( - Sys.getenv("GEOTARGETS_GDAL_RASTER_CREATION_OPTIONS", - unset = ";"), - ";")[[1]] - - for (i in seq(object)) { - if (i > 1) { - opt <- "APPEND_SUBDATASET=YES" - } else { - opt <- "" - } - terra::writeRaster( - x = object[i], - filename = path, - filetype = Sys.getenv("GEOTARGETS_GDAL_RASTER_DRIVER"), - overwrite = (i == 1), - gdal = c(opt, gdal) - ) - } - }, - marshal = function(object) terra::wrap(object), - unmarshal = function(object) terra::unwrap(object) - ) -} +#' +#' +#' #' Format function for sprc and sds +#' #' @noRd +#' format_terra_collections <- function(type = c("sprc", "sds")) { +#' type <- match.arg(type) +#' targets::tar_format( +#' read = switch(type, +#' "sprc" = function(path) terra::sprc(path), +#' "sds" = function(path) terra::sds(path) +#' ), +#' write = function(object, path) { +#' gdal <- strsplit( +#' Sys.getenv("GEOTARGETS_GDAL_RASTER_CREATION_OPTIONS", +#' unset = ";"), +#' ";")[[1]] +#' +#' for (i in seq(object)) { +#' if (i > 1) { +#' opt <- "APPEND_SUBDATASET=YES" +#' } else { +#' opt <- "" +#' } +#' terra::writeRaster( +#' x = object[i], +#' filename = path, +#' filetype = Sys.getenv("GEOTARGETS_GDAL_RASTER_DRIVER"), +#' overwrite = (i == 1), +#' gdal = c(opt, gdal) +#' ) +#' } +#' }, +#' marshal = function(object) terra::wrap(object), +#' unmarshal = function(object) terra::unwrap(object) +#' ) +#' } From 1dcc6b56ead4bf2e37521c66ec270e5a10225309 Mon Sep 17 00:00:00 2001 From: Eric Scott Date: Thu, 10 Oct 2024 09:28:46 -0700 Subject: [PATCH 06/10] use substitute method for tar_terra_tiles --- R/tar_terra_tiles.R | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/R/tar_terra_tiles.R b/R/tar_terra_tiles.R index 6b5c602..4b645b7 100644 --- a/R/tar_terra_tiles.R +++ b/R/tar_terra_tiles.R @@ -168,16 +168,14 @@ tar_terra_tiles_raw <- function( terra::writeRaster( object, path, - filetype = Sys.getenv("GEOTARGETS_GDAL_RASTER_DRIVER"), + filetype = filetype, overwrite = TRUE, - gdal = strsplit( - Sys.getenv("GEOTARGETS_GDAL_RASTER_CREATION_OPTIONS", - unset = ";"), - ";")[[1]] + gdal = gdal ) }, marshal = function(object) terra::wrap(object), - unmarshal = function(object) terra::unwrap(object) + unmarshal = function(object) terra::unwrap(object), + substitute = list(filetype = filetype, gdal = gdal) ), repository = repository, iteration = "list", #only list works (for now at least) @@ -186,18 +184,7 @@ tar_terra_tiles_raw <- function( garbage_collection = garbage_collection, deployment = deployment, priority = priority, - resources = utils::modifyList( - targets::tar_resources( - custom_format = targets::tar_resources_custom_format( - #these envvars are used in write function of format - envvars = c( - "GEOTARGETS_GDAL_RASTER_DRIVER" = filetype, - "GEOTARGETS_GDAL_RASTER_CREATION_OPTIONS" = ( - paste0(gdal, collapse = ";") - ) - ) - ) - ), resources), + resources = resources, storage = storage, retrieval = retrieval, cue = cue, From de1f235356323b379b1ce849b73c7575ea572805 Mon Sep 17 00:00:00 2001 From: Eric Scott Date: Thu, 10 Oct 2024 10:42:17 -0700 Subject: [PATCH 07/10] document() --- man/tar_stars.Rd | 47 ++++++++++++++++++++++++++++++++++-------- man/tar_terra_rast.Rd | 47 ++++++++++++++++++++++++++++++++++-------- man/tar_terra_sds.Rd | 47 ++++++++++++++++++++++++++++++++++-------- man/tar_terra_sprc.Rd | 47 ++++++++++++++++++++++++++++++++++-------- man/tar_terra_tiles.Rd | 29 +++++++++++++++++++++----- man/tar_terra_vect.Rd | 45 +++++++++++++++++++++++++++++++++------- 6 files changed, 213 insertions(+), 49 deletions(-) diff --git a/man/tar_stars.Rd b/man/tar_stars.Rd index 22f0bff..e5db2fd 100644 --- a/man/tar_stars.Rd +++ b/man/tar_stars.Rd @@ -57,8 +57,13 @@ tar_stars_proxy( ) } \arguments{ -\item{name}{Symbol, name of the target. A target -name must be a valid name for a symbol in R, and it +\item{name}{Symbol, name of the target. +In \code{\link[targets:tar_target]{tar_target()}}, \code{name} is an unevaluated symbol, e.g. +\code{tar_target(name = data)}. +In \code{\link[targets:tar_target_raw]{tar_target_raw()}}, \code{name} is a character string, e.g. +\code{tar_target_raw(name = "data")}. + +A target name must be a valid name for a symbol in R, and it must not start with a dot. Subsequent targets can refer to this name symbolically to induce a dependency relationship: e.g. \code{tar_target(downstream_target, f(upstream_target))} is a @@ -73,10 +78,20 @@ You can recover the seed of a completed target with \code{tar_meta(your_target, seed)} and run \code{\link[targets:tar_seed_set]{tar_seed_set()}} on the result to locally recreate the target's initial RNG state.} -\item{command}{R code to run the target.} +\item{command}{R code to run the target. +In \code{\link[targets:tar_target]{tar_target()}}, \code{command} is an unevaluated expression, e.g. +\code{tar_target(command = data)}. +In \code{\link[targets:tar_target_raw]{tar_target_raw()}}, \code{command} is an evaluated expression, e.g. +\code{tar_target_raw(command = quote(data))}.} + +\item{pattern}{Code to define a dynamic branching branching for a target. +In \code{\link[targets:tar_target]{tar_target()}}, \code{pattern} is an unevaluated expression, e.g. +\code{tar_target(pattern = map(data))}. +In \code{\link[targets:tar_target_raw]{tar_target_raw()}}, \code{command} is an evaluated expression, e.g. +\code{tar_target_raw(pattern = quote(map(data)))}. -\item{pattern}{Language to define branching for a target. -For example, in a pipeline with numeric vector targets \code{x} and \code{y}, +To demonstrate dynamic branching patterns, suppose we have +a pipeline with numeric vector targets \code{x} and \code{y}. Then, \code{tar_target(z, x + y, pattern = map(x, y))} implicitly defines branches of \code{z} that each compute \code{x[1] + y[1]}, \code{x[2] + y[2]}, and so on. See the user manual for details.} @@ -121,6 +136,8 @@ for details for instructions. See the cloud storage section of \url{https://books.ropensci.org/targets/data.html} for details for instructions. +\item A character string from \code{\link[targets:tar_repository_cas]{tar_repository_cas()}} for content-addressable +storage. } Note: if \code{repository} is not \code{"local"} and \code{format} is \code{"file"} @@ -134,13 +151,25 @@ stops and throws an error. Options: \itemize{ \item \code{"stop"}: the whole pipeline stops and throws an error. \item \code{"continue"}: the whole pipeline keeps going. +\item \code{"null"}: The errored target continues and returns \code{NULL}. +The data hash is deliberately wrong so the target is not +up to date for the next run of the pipeline. \item \code{"abridge"}: any currently running targets keep running, but no new targets launch after that. +\item \code{"trim"}: all currently running targets stay running. A queued +target is allowed to start if: +\enumerate{ +\item It is not downstream of the error, and +\item It is not a sibling branch from the same \code{\link[targets:tar_target]{tar_target()}} call +(if the error happened in a dynamic branch). +} + +The idea is to avoid starting any new work that the immediate error +impacts. \code{error = "trim"} is just like \code{error = "abridge"}, +but it allows potentially healthy regions of the dependency graph +to begin running. (Visit \url{https://books.ropensci.org/targets/debugging.html} to learn how to debug targets using saved workspaces.) -\item \code{"null"}: The errored target continues and returns \code{NULL}. -The data hash is deliberately wrong so the target is not -up to date for the next run of the pipeline. }} \item{memory}{Character of length 1, memory strategy. @@ -267,5 +296,5 @@ if (Sys.getenv("TAR_LONG_EXAMPLES") == "true") { \dontshow{\}) # examplesIf} } \seealso{ -\code{\link[targets:tar_target_raw]{targets::tar_target_raw()}} +\code{\link[targets:tar_target]{targets::tar_target_raw()}} } diff --git a/man/tar_terra_rast.Rd b/man/tar_terra_rast.Rd index 579970c..198fb1b 100644 --- a/man/tar_terra_rast.Rd +++ b/man/tar_terra_rast.Rd @@ -28,8 +28,13 @@ tar_terra_rast( ) } \arguments{ -\item{name}{Symbol, name of the target. A target -name must be a valid name for a symbol in R, and it +\item{name}{Symbol, name of the target. +In \code{\link[targets:tar_target]{tar_target()}}, \code{name} is an unevaluated symbol, e.g. +\code{tar_target(name = data)}. +In \code{\link[targets:tar_target_raw]{tar_target_raw()}}, \code{name} is a character string, e.g. +\code{tar_target_raw(name = "data")}. + +A target name must be a valid name for a symbol in R, and it must not start with a dot. Subsequent targets can refer to this name symbolically to induce a dependency relationship: e.g. \code{tar_target(downstream_target, f(upstream_target))} is a @@ -44,10 +49,20 @@ You can recover the seed of a completed target with \code{tar_meta(your_target, seed)} and run \code{\link[targets:tar_seed_set]{tar_seed_set()}} on the result to locally recreate the target's initial RNG state.} -\item{command}{R code to run the target.} +\item{command}{R code to run the target. +In \code{\link[targets:tar_target]{tar_target()}}, \code{command} is an unevaluated expression, e.g. +\code{tar_target(command = data)}. +In \code{\link[targets:tar_target_raw]{tar_target_raw()}}, \code{command} is an evaluated expression, e.g. +\code{tar_target_raw(command = quote(data))}.} + +\item{pattern}{Code to define a dynamic branching branching for a target. +In \code{\link[targets:tar_target]{tar_target()}}, \code{pattern} is an unevaluated expression, e.g. +\code{tar_target(pattern = map(data))}. +In \code{\link[targets:tar_target_raw]{tar_target_raw()}}, \code{command} is an evaluated expression, e.g. +\code{tar_target_raw(pattern = quote(map(data)))}. -\item{pattern}{Language to define branching for a target. -For example, in a pipeline with numeric vector targets \code{x} and \code{y}, +To demonstrate dynamic branching patterns, suppose we have +a pipeline with numeric vector targets \code{x} and \code{y}. Then, \code{tar_target(z, x + y, pattern = map(x, y))} implicitly defines branches of \code{z} that each compute \code{x[1] + y[1]}, \code{x[2] + y[2]}, and so on. See the user manual for details.} @@ -88,6 +103,8 @@ for details for instructions. See the cloud storage section of \url{https://books.ropensci.org/targets/data.html} for details for instructions. +\item A character string from \code{\link[targets:tar_repository_cas]{tar_repository_cas()}} for content-addressable +storage. } Note: if \code{repository} is not \code{"local"} and \code{format} is \code{"file"} @@ -101,13 +118,25 @@ stops and throws an error. Options: \itemize{ \item \code{"stop"}: the whole pipeline stops and throws an error. \item \code{"continue"}: the whole pipeline keeps going. +\item \code{"null"}: The errored target continues and returns \code{NULL}. +The data hash is deliberately wrong so the target is not +up to date for the next run of the pipeline. \item \code{"abridge"}: any currently running targets keep running, but no new targets launch after that. +\item \code{"trim"}: all currently running targets stay running. A queued +target is allowed to start if: +\enumerate{ +\item It is not downstream of the error, and +\item It is not a sibling branch from the same \code{\link[targets:tar_target]{tar_target()}} call +(if the error happened in a dynamic branch). +} + +The idea is to avoid starting any new work that the immediate error +impacts. \code{error = "trim"} is just like \code{error = "abridge"}, +but it allows potentially healthy regions of the dependency graph +to begin running. (Visit \url{https://books.ropensci.org/targets/debugging.html} to learn how to debug targets using saved workspaces.) -\item \code{"null"}: The errored target continues and returns \code{NULL}. -The data hash is deliberately wrong so the target is not -up to date for the next run of the pipeline. }} \item{memory}{Character of length 1, memory strategy. @@ -235,5 +264,5 @@ if (Sys.getenv("TAR_LONG_EXAMPLES") == "true") { } } \seealso{ -\code{\link[targets:tar_target_raw]{targets::tar_target_raw()}} +\code{\link[targets:tar_target]{targets::tar_target_raw()}} } diff --git a/man/tar_terra_sds.Rd b/man/tar_terra_sds.Rd index 35bb05c..aa7047b 100644 --- a/man/tar_terra_sds.Rd +++ b/man/tar_terra_sds.Rd @@ -28,8 +28,13 @@ tar_terra_sds( ) } \arguments{ -\item{name}{Symbol, name of the target. A target -name must be a valid name for a symbol in R, and it +\item{name}{Symbol, name of the target. +In \code{\link[targets:tar_target]{tar_target()}}, \code{name} is an unevaluated symbol, e.g. +\code{tar_target(name = data)}. +In \code{\link[targets:tar_target_raw]{tar_target_raw()}}, \code{name} is a character string, e.g. +\code{tar_target_raw(name = "data")}. + +A target name must be a valid name for a symbol in R, and it must not start with a dot. Subsequent targets can refer to this name symbolically to induce a dependency relationship: e.g. \code{tar_target(downstream_target, f(upstream_target))} is a @@ -44,10 +49,20 @@ You can recover the seed of a completed target with \code{tar_meta(your_target, seed)} and run \code{\link[targets:tar_seed_set]{tar_seed_set()}} on the result to locally recreate the target's initial RNG state.} -\item{command}{R code to run the target.} +\item{command}{R code to run the target. +In \code{\link[targets:tar_target]{tar_target()}}, \code{command} is an unevaluated expression, e.g. +\code{tar_target(command = data)}. +In \code{\link[targets:tar_target_raw]{tar_target_raw()}}, \code{command} is an evaluated expression, e.g. +\code{tar_target_raw(command = quote(data))}.} + +\item{pattern}{Code to define a dynamic branching branching for a target. +In \code{\link[targets:tar_target]{tar_target()}}, \code{pattern} is an unevaluated expression, e.g. +\code{tar_target(pattern = map(data))}. +In \code{\link[targets:tar_target_raw]{tar_target_raw()}}, \code{command} is an evaluated expression, e.g. +\code{tar_target_raw(pattern = quote(map(data)))}. -\item{pattern}{Language to define branching for a target. -For example, in a pipeline with numeric vector targets \code{x} and \code{y}, +To demonstrate dynamic branching patterns, suppose we have +a pipeline with numeric vector targets \code{x} and \code{y}. Then, \code{tar_target(z, x + y, pattern = map(x, y))} implicitly defines branches of \code{z} that each compute \code{x[1] + y[1]}, \code{x[2] + y[2]}, and so on. See the user manual for details.} @@ -88,6 +103,8 @@ for details for instructions. See the cloud storage section of \url{https://books.ropensci.org/targets/data.html} for details for instructions. +\item A character string from \code{\link[targets:tar_repository_cas]{tar_repository_cas()}} for content-addressable +storage. } Note: if \code{repository} is not \code{"local"} and \code{format} is \code{"file"} @@ -101,13 +118,25 @@ stops and throws an error. Options: \itemize{ \item \code{"stop"}: the whole pipeline stops and throws an error. \item \code{"continue"}: the whole pipeline keeps going. +\item \code{"null"}: The errored target continues and returns \code{NULL}. +The data hash is deliberately wrong so the target is not +up to date for the next run of the pipeline. \item \code{"abridge"}: any currently running targets keep running, but no new targets launch after that. +\item \code{"trim"}: all currently running targets stay running. A queued +target is allowed to start if: +\enumerate{ +\item It is not downstream of the error, and +\item It is not a sibling branch from the same \code{\link[targets:tar_target]{tar_target()}} call +(if the error happened in a dynamic branch). +} + +The idea is to avoid starting any new work that the immediate error +impacts. \code{error = "trim"} is just like \code{error = "abridge"}, +but it allows potentially healthy regions of the dependency graph +to begin running. (Visit \url{https://books.ropensci.org/targets/debugging.html} to learn how to debug targets using saved workspaces.) -\item \code{"null"}: The errored target continues and returns \code{NULL}. -The data hash is deliberately wrong so the target is not -up to date for the next run of the pipeline. }} \item{memory}{Character of length 1, memory strategy. @@ -243,7 +272,7 @@ if (Sys.getenv("TAR_LONG_EXAMPLES") == "true") { } } \seealso{ -\code{\link[targets:tar_target_raw]{targets::tar_target_raw()}}, \code{\link[=tar_terra_sprc]{tar_terra_sprc()}} +\code{\link[targets:tar_target]{targets::tar_target_raw()}}, \code{\link[=tar_terra_sprc]{tar_terra_sprc()}} } \author{ Andrew Gene Brown diff --git a/man/tar_terra_sprc.Rd b/man/tar_terra_sprc.Rd index bb063f4..49674e6 100644 --- a/man/tar_terra_sprc.Rd +++ b/man/tar_terra_sprc.Rd @@ -28,8 +28,13 @@ tar_terra_sprc( ) } \arguments{ -\item{name}{Symbol, name of the target. A target -name must be a valid name for a symbol in R, and it +\item{name}{Symbol, name of the target. +In \code{\link[targets:tar_target]{tar_target()}}, \code{name} is an unevaluated symbol, e.g. +\code{tar_target(name = data)}. +In \code{\link[targets:tar_target_raw]{tar_target_raw()}}, \code{name} is a character string, e.g. +\code{tar_target_raw(name = "data")}. + +A target name must be a valid name for a symbol in R, and it must not start with a dot. Subsequent targets can refer to this name symbolically to induce a dependency relationship: e.g. \code{tar_target(downstream_target, f(upstream_target))} is a @@ -44,10 +49,20 @@ You can recover the seed of a completed target with \code{tar_meta(your_target, seed)} and run \code{\link[targets:tar_seed_set]{tar_seed_set()}} on the result to locally recreate the target's initial RNG state.} -\item{command}{R code to run the target.} +\item{command}{R code to run the target. +In \code{\link[targets:tar_target]{tar_target()}}, \code{command} is an unevaluated expression, e.g. +\code{tar_target(command = data)}. +In \code{\link[targets:tar_target_raw]{tar_target_raw()}}, \code{command} is an evaluated expression, e.g. +\code{tar_target_raw(command = quote(data))}.} + +\item{pattern}{Code to define a dynamic branching branching for a target. +In \code{\link[targets:tar_target]{tar_target()}}, \code{pattern} is an unevaluated expression, e.g. +\code{tar_target(pattern = map(data))}. +In \code{\link[targets:tar_target_raw]{tar_target_raw()}}, \code{command} is an evaluated expression, e.g. +\code{tar_target_raw(pattern = quote(map(data)))}. -\item{pattern}{Language to define branching for a target. -For example, in a pipeline with numeric vector targets \code{x} and \code{y}, +To demonstrate dynamic branching patterns, suppose we have +a pipeline with numeric vector targets \code{x} and \code{y}. Then, \code{tar_target(z, x + y, pattern = map(x, y))} implicitly defines branches of \code{z} that each compute \code{x[1] + y[1]}, \code{x[2] + y[2]}, and so on. See the user manual for details.} @@ -88,6 +103,8 @@ for details for instructions. See the cloud storage section of \url{https://books.ropensci.org/targets/data.html} for details for instructions. +\item A character string from \code{\link[targets:tar_repository_cas]{tar_repository_cas()}} for content-addressable +storage. } Note: if \code{repository} is not \code{"local"} and \code{format} is \code{"file"} @@ -101,13 +118,25 @@ stops and throws an error. Options: \itemize{ \item \code{"stop"}: the whole pipeline stops and throws an error. \item \code{"continue"}: the whole pipeline keeps going. +\item \code{"null"}: The errored target continues and returns \code{NULL}. +The data hash is deliberately wrong so the target is not +up to date for the next run of the pipeline. \item \code{"abridge"}: any currently running targets keep running, but no new targets launch after that. +\item \code{"trim"}: all currently running targets stay running. A queued +target is allowed to start if: +\enumerate{ +\item It is not downstream of the error, and +\item It is not a sibling branch from the same \code{\link[targets:tar_target]{tar_target()}} call +(if the error happened in a dynamic branch). +} + +The idea is to avoid starting any new work that the immediate error +impacts. \code{error = "trim"} is just like \code{error = "abridge"}, +but it allows potentially healthy regions of the dependency graph +to begin running. (Visit \url{https://books.ropensci.org/targets/debugging.html} to learn how to debug targets using saved workspaces.) -\item \code{"null"}: The errored target continues and returns \code{NULL}. -The data hash is deliberately wrong so the target is not -up to date for the next run of the pipeline. }} \item{memory}{Character of length 1, memory strategy. @@ -247,7 +276,7 @@ if (Sys.getenv("TAR_LONG_EXAMPLES") == "true") { } } \seealso{ -\code{\link[targets:tar_target_raw]{targets::tar_target_raw()}} +\code{\link[targets:tar_target]{targets::tar_target_raw()}} } \author{ Andrew Gene Brown diff --git a/man/tar_terra_tiles.Rd b/man/tar_terra_tiles.Rd index 4ee8f47..042c00a 100644 --- a/man/tar_terra_tiles.Rd +++ b/man/tar_terra_tiles.Rd @@ -27,8 +27,13 @@ tar_terra_tiles( ) } \arguments{ -\item{name}{Symbol, name of the target. A target -name must be a valid name for a symbol in R, and it +\item{name}{Symbol, name of the target. +In \code{\link[targets:tar_target]{tar_target()}}, \code{name} is an unevaluated symbol, e.g. +\code{tar_target(name = data)}. +In \code{\link[targets:tar_target_raw]{tar_target_raw()}}, \code{name} is a character string, e.g. +\code{tar_target_raw(name = "data")}. + +A target name must be a valid name for a symbol in R, and it must not start with a dot. Subsequent targets can refer to this name symbolically to induce a dependency relationship: e.g. \code{tar_target(downstream_target, f(upstream_target))} is a @@ -82,6 +87,8 @@ for details for instructions. See the cloud storage section of \url{https://books.ropensci.org/targets/data.html} for details for instructions. +\item A character string from \code{\link[targets:tar_repository_cas]{tar_repository_cas()}} for content-addressable +storage. } Note: if \code{repository} is not \code{"local"} and \code{format} is \code{"file"} @@ -95,13 +102,25 @@ stops and throws an error. Options: \itemize{ \item \code{"stop"}: the whole pipeline stops and throws an error. \item \code{"continue"}: the whole pipeline keeps going. +\item \code{"null"}: The errored target continues and returns \code{NULL}. +The data hash is deliberately wrong so the target is not +up to date for the next run of the pipeline. \item \code{"abridge"}: any currently running targets keep running, but no new targets launch after that. +\item \code{"trim"}: all currently running targets stay running. A queued +target is allowed to start if: +\enumerate{ +\item It is not downstream of the error, and +\item It is not a sibling branch from the same \code{\link[targets:tar_target]{tar_target()}} call +(if the error happened in a dynamic branch). +} + +The idea is to avoid starting any new work that the immediate error +impacts. \code{error = "trim"} is just like \code{error = "abridge"}, +but it allows potentially healthy regions of the dependency graph +to begin running. (Visit \url{https://books.ropensci.org/targets/debugging.html} to learn how to debug targets using saved workspaces.) -\item \code{"null"}: The errored target continues and returns \code{NULL}. -The data hash is deliberately wrong so the target is not -up to date for the next run of the pipeline. }} \item{memory}{Character of length 1, memory strategy. diff --git a/man/tar_terra_vect.Rd b/man/tar_terra_vect.Rd index 2266761..263b983 100644 --- a/man/tar_terra_vect.Rd +++ b/man/tar_terra_vect.Rd @@ -28,8 +28,13 @@ tar_terra_vect( ) } \arguments{ -\item{name}{Symbol, name of the target. A target -name must be a valid name for a symbol in R, and it +\item{name}{Symbol, name of the target. +In \code{\link[targets:tar_target]{tar_target()}}, \code{name} is an unevaluated symbol, e.g. +\code{tar_target(name = data)}. +In \code{\link[targets:tar_target_raw]{tar_target_raw()}}, \code{name} is a character string, e.g. +\code{tar_target_raw(name = "data")}. + +A target name must be a valid name for a symbol in R, and it must not start with a dot. Subsequent targets can refer to this name symbolically to induce a dependency relationship: e.g. \code{tar_target(downstream_target, f(upstream_target))} is a @@ -44,10 +49,20 @@ You can recover the seed of a completed target with \code{tar_meta(your_target, seed)} and run \code{\link[targets:tar_seed_set]{tar_seed_set()}} on the result to locally recreate the target's initial RNG state.} -\item{command}{R code to run the target.} +\item{command}{R code to run the target. +In \code{\link[targets:tar_target]{tar_target()}}, \code{command} is an unevaluated expression, e.g. +\code{tar_target(command = data)}. +In \code{\link[targets:tar_target_raw]{tar_target_raw()}}, \code{command} is an evaluated expression, e.g. +\code{tar_target_raw(command = quote(data))}.} + +\item{pattern}{Code to define a dynamic branching branching for a target. +In \code{\link[targets:tar_target]{tar_target()}}, \code{pattern} is an unevaluated expression, e.g. +\code{tar_target(pattern = map(data))}. +In \code{\link[targets:tar_target_raw]{tar_target_raw()}}, \code{command} is an evaluated expression, e.g. +\code{tar_target_raw(pattern = quote(map(data)))}. -\item{pattern}{Language to define branching for a target. -For example, in a pipeline with numeric vector targets \code{x} and \code{y}, +To demonstrate dynamic branching patterns, suppose we have +a pipeline with numeric vector targets \code{x} and \code{y}. Then, \code{tar_target(z, x + y, pattern = map(x, y))} implicitly defines branches of \code{z} that each compute \code{x[1] + y[1]}, \code{x[2] + y[2]}, and so on. See the user manual for details.} @@ -88,6 +103,8 @@ for details for instructions. See the cloud storage section of \url{https://books.ropensci.org/targets/data.html} for details for instructions. +\item A character string from \code{\link[targets:tar_repository_cas]{tar_repository_cas()}} for content-addressable +storage. } Note: if \code{repository} is not \code{"local"} and \code{format} is \code{"file"} @@ -101,13 +118,25 @@ stops and throws an error. Options: \itemize{ \item \code{"stop"}: the whole pipeline stops and throws an error. \item \code{"continue"}: the whole pipeline keeps going. +\item \code{"null"}: The errored target continues and returns \code{NULL}. +The data hash is deliberately wrong so the target is not +up to date for the next run of the pipeline. \item \code{"abridge"}: any currently running targets keep running, but no new targets launch after that. +\item \code{"trim"}: all currently running targets stay running. A queued +target is allowed to start if: +\enumerate{ +\item It is not downstream of the error, and +\item It is not a sibling branch from the same \code{\link[targets:tar_target]{tar_target()}} call +(if the error happened in a dynamic branch). +} + +The idea is to avoid starting any new work that the immediate error +impacts. \code{error = "trim"} is just like \code{error = "abridge"}, +but it allows potentially healthy regions of the dependency graph +to begin running. (Visit \url{https://books.ropensci.org/targets/debugging.html} to learn how to debug targets using saved workspaces.) -\item \code{"null"}: The errored target continues and returns \code{NULL}. -The data hash is deliberately wrong so the target is not -up to date for the next run of the pipeline. }} \item{memory}{Character of length 1, memory strategy. From 49ff96055fe618814c1fcc22bb10ae59b6b38ad9 Mon Sep 17 00:00:00 2001 From: Eric Scott Date: Fri, 11 Oct 2024 09:38:26 -0700 Subject: [PATCH 08/10] use check_pkg_installed --- R/tar-stars.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/R/tar-stars.R b/R/tar-stars.R index 6dee703..f37cb4a 100644 --- a/R/tar-stars.R +++ b/R/tar-stars.R @@ -212,7 +212,8 @@ tar_stars_raw <- function(name, library = library, format = targets::tar_format( read = function(path) { - if (ncdf && requireNamespace("ncmeta")) { + if (ncdf) { + check_pkg_installed("ncmeta") stars::read_ncdf(path, proxy = proxy) } else if (isTRUE(mdim)) { stars::read_mdim(path, proxy = proxy) From bb41d241f8211b8fbcafc175d0308def0274ab15 Mon Sep 17 00:00:00 2001 From: Eric Scott Date: Fri, 11 Oct 2024 09:51:31 -0700 Subject: [PATCH 09/10] need geotargets::: to reach unexported funs from inside tar_format() --- R/tar-stars.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/tar-stars.R b/R/tar-stars.R index f37cb4a..52fd2d6 100644 --- a/R/tar-stars.R +++ b/R/tar-stars.R @@ -213,7 +213,7 @@ tar_stars_raw <- function(name, format = targets::tar_format( read = function(path) { if (ncdf) { - check_pkg_installed("ncmeta") + geotargets:::check_pkg_installed("ncmeta") stars::read_ncdf(path, proxy = proxy) } else if (isTRUE(mdim)) { stars::read_mdim(path, proxy = proxy) From dc5750171cb8bf7b70d097203285f6baec3fc699 Mon Sep 17 00:00:00 2001 From: Eric Scott Date: Fri, 11 Oct 2024 12:20:16 -0700 Subject: [PATCH 10/10] use tar_outdated() in test --- tests/testthat/test-tar-terra.R | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/testthat/test-tar-terra.R b/tests/testthat/test-tar-terra.R index f31f956..136a16b 100644 --- a/tests/testthat/test-tar-terra.R +++ b/tests/testthat/test-tar-terra.R @@ -181,7 +181,7 @@ tar_test("That changing filetype invalidates a target", { ) }) tar_make() - first <- tar_meta()$time[2] + targets::tar_script({ library(targets) library(geotargets) @@ -195,9 +195,5 @@ tar_test("That changing filetype invalidates a target", { ) ) }) - tar_make() - second <- tar_meta()$time[2] - #There's gotta be a better way to test if a target is invalidated - expect_gt(second, first) - + expect_equal(tar_outdated(), "r") })