From 115783d1b881e72e8b3bf79f3ddf38b792e73ffc Mon Sep 17 00:00:00 2001 From: Andrew Gene Brown Date: Thu, 28 Nov 2024 11:29:49 -0800 Subject: [PATCH 01/12] tar_terra_rast: implement gdalraster SOZip `preserve_metadata` method --- R/tar-terra-rast.R | 47 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/R/tar-terra-rast.R b/R/tar-terra-rast.R index 0461ceb..f8d7221 100644 --- a/R/tar-terra-rast.R +++ b/R/tar-terra-rast.R @@ -80,15 +80,19 @@ tar_terra_rast <- function(name, filetype <- filetype %||% "GTiff" - #check that filetype option is available + # check that filetype option is available drv <- get_gdal_available_driver_list("raster") filetype <- rlang::arg_match0(filetype, drv$name) - #currently only "drop" and "zip" are valid options + # various methods for packaging geospatial data and auxiliary files preserve_metadata <- preserve_metadata %||% "drop" - preserve_metadata <- rlang::arg_match0(preserve_metadata, c("drop", "zip")) + preserve_metadata <- rlang::arg_match0(preserve_metadata, c("drop", "zip", "gdalraster_sozip")) - #ensure that user-passed `resources` doesn't include `custom_format` + if (preserve_metadata == "gdalraster_sozip") { + check_pkg_installed("gdalraster") + } + + # ensure that user-passed `resources` doesn't include `custom_format` if ("custom_format" %in% names(resources)) { cli::cli_abort("{.val custom_format} cannot be supplied to targets created with {.fn tar_terra_rast}") } @@ -145,6 +149,9 @@ tar_rast_read <- function(preserve_metadata) { zip::unzip(zipfile = path, exdir = tmp) terra::rast(file.path(tmp, basename(path))) }, + gdalraster_sozip = function(path) { + terra::rast(paste0("/vsizip/{", path, "}/"), basename(path)) + }, drop = function(path) terra::rast(path) ) } @@ -175,6 +182,38 @@ tar_rast_write <- function(filetype, gdal, preserve_metadata) { #move the zip file to the expected place file.rename(file.path(tmp, basename(path)), path) }, + gdalraster_sozip = function(object, path) { + + tmp <- withr::local_tempdir() + + dir.create(file.path(tmp, dirname(path)), recursive = TRUE) + + terra::writeRaster( + object, + file.path(tmp, path), + filetype = filetype, + overwrite = TRUE, + gdal = gdal + ) + + raster_files <- list.files(file.path(tmp, dirname(path)), full.names = TRUE) + + # create seek-optimized zip file using gdalraster + gdalraster::addFilesInZip( + path, + raster_files, + full_paths = FALSE, + overwrite = TRUE, + sozip_enabled = "YES", + num_threads = 1, + quiet = TRUE + ) + # always create sozip regardless of file size (sozip_enabled = "YES") + # TODO: allow user control of number of threads? + # how does num_threads interact multiple workers etc.? + + unlink(file.path(tmp, path), ) + }, drop = function(object, path) { terra::writeRaster( object, From a948b009685aff52879dcca9352df078bc31f72f Mon Sep 17 00:00:00 2001 From: Andrew Gene Brown Date: Thu, 28 Nov 2024 12:10:35 -0800 Subject: [PATCH 02/12] add gdalraster to suggests --- DESCRIPTION | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 91689c5..6272e02 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -47,7 +47,8 @@ Suggests: sf, stars, testthat (>= 3.0.0), - fs + fs, + gdalraster Config/testthat/edition: 3 URL: https://github.com/njtierney/geotargets, https://njtierney.github.io/geotargets/ BugReports: https://github.com/njtierney/geotargets/issues From f4b7a267ab8b789f6b67044ae70d0876a0ad19bd Mon Sep 17 00:00:00 2001 From: Andrew Gene Brown Date: Mon, 2 Dec 2024 06:52:22 -0800 Subject: [PATCH 03/12] variable for temp file path --- R/tar-terra-rast.R | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/R/tar-terra-rast.R b/R/tar-terra-rast.R index 0f3113f..c93d361 100644 --- a/R/tar-terra-rast.R +++ b/R/tar-terra-rast.R @@ -89,7 +89,7 @@ tar_terra_rast <- function(name, # ensure that user-passed `resources` doesn't include `custom_format` check_user_resources(resources) - + if (preserve_metadata == "gdalraster_sozip") { check_pkg_installed("gdalraster") } @@ -164,7 +164,8 @@ tar_rast_write <- function(filetype, gdal, preserve_metadata) { zip = function(object, path) { #write the raster in a fresh local tempdir() that disappears when function is done tmp <- withr::local_tempdir() - dir.create(file.path(tmp, dirname(path)), recursive = TRUE) + tmppath <- file.path(tmp, dirname(path)) + dir.create(tmppath, recursive = TRUE) terra::writeRaster( object, file.path(tmp, path), @@ -173,7 +174,7 @@ tar_rast_write <- function(filetype, gdal, preserve_metadata) { gdal = gdal ) #package files into a zip file using `zip::zip()` - raster_files <- list.files(file.path(tmp, dirname(path)), full.names = TRUE) + raster_files <- list.files(tmppath, full.names = TRUE) zip::zip( file.path(tmp, basename(path)), files = raster_files, @@ -183,14 +184,14 @@ tar_rast_write <- function(filetype, gdal, preserve_metadata) { ) # move the zip file to the expected place - file.copy(file.path(tmp, basename(path)), path) - unlink(file.path(tmp, basename(path))) + file.copy(tmppath, path) + unlink(tmppath) }, gdalraster_sozip = function(object, path) { tmp <- withr::local_tempdir() - - dir.create(file.path(tmp, dirname(path)), recursive = TRUE) + tmppath <- file.path(tmp, dirname(path)) + dir.create(tmppath, recursive = TRUE) terra::writeRaster( object, @@ -200,7 +201,7 @@ tar_rast_write <- function(filetype, gdal, preserve_metadata) { gdal = gdal ) - raster_files <- list.files(file.path(tmp, dirname(path)), full.names = TRUE) + raster_files <- list.files(tmppath, full.names = TRUE) # create seek-optimized zip file using gdalraster gdalraster::addFilesInZip( @@ -216,7 +217,7 @@ tar_rast_write <- function(filetype, gdal, preserve_metadata) { # TODO: allow user control of number of threads? # how does num_threads interact multiple workers etc.? - unlink(file.path(tmp, path)) + unlink(tmppath) }, drop = function(object, path) { terra::writeRaster( From 3413763ef7b77f1af437149699ff46784b130314 Mon Sep 17 00:00:00 2001 From: Andrew Gene Brown Date: Mon, 2 Dec 2024 07:01:44 -0800 Subject: [PATCH 04/12] whitespace --- R/tar-terra-rast.R | 129 ++++++++++++++++++++++----------------------- 1 file changed, 64 insertions(+), 65 deletions(-) diff --git a/R/tar-terra-rast.R b/R/tar-terra-rast.R index c93d361..da304c1 100644 --- a/R/tar-terra-rast.R +++ b/R/tar-terra-rast.R @@ -159,74 +159,73 @@ tar_rast_read <- function(preserve_metadata) { } tar_rast_write <- function(filetype, gdal, preserve_metadata) { - switch( - preserve_metadata, - zip = function(object, path) { - #write the raster in a fresh local tempdir() that disappears when function is done - tmp <- withr::local_tempdir() - tmppath <- file.path(tmp, dirname(path)) - dir.create(tmppath, recursive = TRUE) - terra::writeRaster( - object, - file.path(tmp, path), - filetype = filetype, - overwrite = TRUE, - gdal = gdal - ) - #package files into a zip file using `zip::zip()` - raster_files <- list.files(tmppath, full.names = TRUE) - zip::zip( - file.path(tmp, basename(path)), - files = raster_files, - compression_level = 1, - mode = "cherry-pick", - root = dirname(raster_files)[1] - ) + switch( + preserve_metadata, + zip = function(object, path) { + #write the raster in a fresh local tempdir() that disappears when function is done + tmp <- withr::local_tempdir() + tmppath <- file.path(tmp, dirname(path)) + dir.create(tmppath, recursive = TRUE) + terra::writeRaster( + object, + file.path(tmp, path), + filetype = filetype, + overwrite = TRUE, + gdal = gdal + ) + #package files into a zip file using `zip::zip()` + raster_files <- list.files(tmppath, full.names = TRUE) + zip::zip( + file.path(tmp, basename(path)), + files = raster_files, + compression_level = 1, + mode = "cherry-pick", + root = dirname(raster_files)[1] + ) # move the zip file to the expected place file.copy(tmppath, path) unlink(tmppath) - }, - gdalraster_sozip = function(object, path) { + }, + gdalraster_sozip = function(object, path) { + tmp <- withr::local_tempdir() + tmppath <- file.path(tmp, dirname(path)) + dir.create(tmppath, recursive = TRUE) + + terra::writeRaster( + object, + file.path(tmp, path), + filetype = filetype, + overwrite = TRUE, + gdal = gdal + ) + + raster_files <- list.files(tmppath, full.names = TRUE) + + # create seek-optimized zip file using gdalraster + gdalraster::addFilesInZip( + path, + raster_files, + full_paths = FALSE, + overwrite = TRUE, + sozip_enabled = "YES", + num_threads = 1, + quiet = TRUE + ) + # always create sozip regardless of file size (sozip_enabled = "YES") + # TODO: allow user control of number of threads? + # how does num_threads interact multiple workers etc.? - tmp <- withr::local_tempdir() - tmppath <- file.path(tmp, dirname(path)) - dir.create(tmppath, recursive = TRUE) - - terra::writeRaster( - object, - file.path(tmp, path), - filetype = filetype, - overwrite = TRUE, - gdal = gdal - ) - - raster_files <- list.files(tmppath, full.names = TRUE) - - # create seek-optimized zip file using gdalraster - gdalraster::addFilesInZip( - path, - raster_files, - full_paths = FALSE, - overwrite = TRUE, - sozip_enabled = "YES", - num_threads = 1, - quiet = TRUE - ) - # always create sozip regardless of file size (sozip_enabled = "YES") - # TODO: allow user control of number of threads? - # how does num_threads interact multiple workers etc.? - - unlink(tmppath) - }, - drop = function(object, path) { - terra::writeRaster( - object, - path, - filetype = filetype, - overwrite = TRUE, - gdal = gdal - ) - } - ) + unlink(tmppath) + }, + drop = function(object, path) { + terra::writeRaster( + object, + path, + filetype = filetype, + overwrite = TRUE, + gdal = gdal + ) + } + ) } From 3532b019b31b46519c2b54621e2743005a07bc5f Mon Sep 17 00:00:00 2001 From: Andrew Gene Brown Date: Mon, 2 Dec 2024 07:24:23 -0800 Subject: [PATCH 05/12] Revert "whitespace" This reverts commit 3413763ef7b77f1af437149699ff46784b130314. --- R/tar-terra-rast.R | 129 +++++++++++++++++++++++---------------------- 1 file changed, 65 insertions(+), 64 deletions(-) diff --git a/R/tar-terra-rast.R b/R/tar-terra-rast.R index da304c1..c93d361 100644 --- a/R/tar-terra-rast.R +++ b/R/tar-terra-rast.R @@ -159,73 +159,74 @@ tar_rast_read <- function(preserve_metadata) { } tar_rast_write <- function(filetype, gdal, preserve_metadata) { - switch( - preserve_metadata, - zip = function(object, path) { - #write the raster in a fresh local tempdir() that disappears when function is done - tmp <- withr::local_tempdir() - tmppath <- file.path(tmp, dirname(path)) - dir.create(tmppath, recursive = TRUE) - terra::writeRaster( - object, - file.path(tmp, path), - filetype = filetype, - overwrite = TRUE, - gdal = gdal - ) - #package files into a zip file using `zip::zip()` - raster_files <- list.files(tmppath, full.names = TRUE) - zip::zip( - file.path(tmp, basename(path)), - files = raster_files, - compression_level = 1, - mode = "cherry-pick", - root = dirname(raster_files)[1] - ) + switch( + preserve_metadata, + zip = function(object, path) { + #write the raster in a fresh local tempdir() that disappears when function is done + tmp <- withr::local_tempdir() + tmppath <- file.path(tmp, dirname(path)) + dir.create(tmppath, recursive = TRUE) + terra::writeRaster( + object, + file.path(tmp, path), + filetype = filetype, + overwrite = TRUE, + gdal = gdal + ) + #package files into a zip file using `zip::zip()` + raster_files <- list.files(tmppath, full.names = TRUE) + zip::zip( + file.path(tmp, basename(path)), + files = raster_files, + compression_level = 1, + mode = "cherry-pick", + root = dirname(raster_files)[1] + ) # move the zip file to the expected place file.copy(tmppath, path) unlink(tmppath) - }, - gdalraster_sozip = function(object, path) { - tmp <- withr::local_tempdir() - tmppath <- file.path(tmp, dirname(path)) - dir.create(tmppath, recursive = TRUE) - - terra::writeRaster( - object, - file.path(tmp, path), - filetype = filetype, - overwrite = TRUE, - gdal = gdal - ) - - raster_files <- list.files(tmppath, full.names = TRUE) - - # create seek-optimized zip file using gdalraster - gdalraster::addFilesInZip( - path, - raster_files, - full_paths = FALSE, - overwrite = TRUE, - sozip_enabled = "YES", - num_threads = 1, - quiet = TRUE - ) - # always create sozip regardless of file size (sozip_enabled = "YES") - # TODO: allow user control of number of threads? - # how does num_threads interact multiple workers etc.? + }, + gdalraster_sozip = function(object, path) { - unlink(tmppath) - }, - drop = function(object, path) { - terra::writeRaster( - object, - path, - filetype = filetype, - overwrite = TRUE, - gdal = gdal - ) - } - ) + tmp <- withr::local_tempdir() + tmppath <- file.path(tmp, dirname(path)) + dir.create(tmppath, recursive = TRUE) + + terra::writeRaster( + object, + file.path(tmp, path), + filetype = filetype, + overwrite = TRUE, + gdal = gdal + ) + + raster_files <- list.files(tmppath, full.names = TRUE) + + # create seek-optimized zip file using gdalraster + gdalraster::addFilesInZip( + path, + raster_files, + full_paths = FALSE, + overwrite = TRUE, + sozip_enabled = "YES", + num_threads = 1, + quiet = TRUE + ) + # always create sozip regardless of file size (sozip_enabled = "YES") + # TODO: allow user control of number of threads? + # how does num_threads interact multiple workers etc.? + + unlink(tmppath) + }, + drop = function(object, path) { + terra::writeRaster( + object, + path, + filetype = filetype, + overwrite = TRUE, + gdal = gdal + ) + } + ) } From 6f092b803e85ee08f786d9792f4aadf97ddf6022 Mon Sep 17 00:00:00 2001 From: Andrew Gene Brown Date: Mon, 2 Dec 2024 07:24:45 -0800 Subject: [PATCH 06/12] Revert "variable for temp file path" This reverts commit f4b7a267ab8b789f6b67044ae70d0876a0ad19bd. --- R/tar-terra-rast.R | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/R/tar-terra-rast.R b/R/tar-terra-rast.R index c93d361..0f3113f 100644 --- a/R/tar-terra-rast.R +++ b/R/tar-terra-rast.R @@ -89,7 +89,7 @@ tar_terra_rast <- function(name, # ensure that user-passed `resources` doesn't include `custom_format` check_user_resources(resources) - + if (preserve_metadata == "gdalraster_sozip") { check_pkg_installed("gdalraster") } @@ -164,8 +164,7 @@ tar_rast_write <- function(filetype, gdal, preserve_metadata) { zip = function(object, path) { #write the raster in a fresh local tempdir() that disappears when function is done tmp <- withr::local_tempdir() - tmppath <- file.path(tmp, dirname(path)) - dir.create(tmppath, recursive = TRUE) + dir.create(file.path(tmp, dirname(path)), recursive = TRUE) terra::writeRaster( object, file.path(tmp, path), @@ -174,7 +173,7 @@ tar_rast_write <- function(filetype, gdal, preserve_metadata) { gdal = gdal ) #package files into a zip file using `zip::zip()` - raster_files <- list.files(tmppath, full.names = TRUE) + raster_files <- list.files(file.path(tmp, dirname(path)), full.names = TRUE) zip::zip( file.path(tmp, basename(path)), files = raster_files, @@ -184,14 +183,14 @@ tar_rast_write <- function(filetype, gdal, preserve_metadata) { ) # move the zip file to the expected place - file.copy(tmppath, path) - unlink(tmppath) + file.copy(file.path(tmp, basename(path)), path) + unlink(file.path(tmp, basename(path))) }, gdalraster_sozip = function(object, path) { tmp <- withr::local_tempdir() - tmppath <- file.path(tmp, dirname(path)) - dir.create(tmppath, recursive = TRUE) + + dir.create(file.path(tmp, dirname(path)), recursive = TRUE) terra::writeRaster( object, @@ -201,7 +200,7 @@ tar_rast_write <- function(filetype, gdal, preserve_metadata) { gdal = gdal ) - raster_files <- list.files(tmppath, full.names = TRUE) + raster_files <- list.files(file.path(tmp, dirname(path)), full.names = TRUE) # create seek-optimized zip file using gdalraster gdalraster::addFilesInZip( @@ -217,7 +216,7 @@ tar_rast_write <- function(filetype, gdal, preserve_metadata) { # TODO: allow user control of number of threads? # how does num_threads interact multiple workers etc.? - unlink(tmppath) + unlink(file.path(tmp, path)) }, drop = function(object, path) { terra::writeRaster( From dd5210a1e6c214b11ea0f05a552e0e00650dccab Mon Sep 17 00:00:00 2001 From: Andrew Gene Brown Date: Mon, 2 Dec 2024 07:33:07 -0800 Subject: [PATCH 07/12] temp paths for gdalraster_sozip method only --- R/tar-terra-rast.R | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/R/tar-terra-rast.R b/R/tar-terra-rast.R index 0f3113f..3ffd4a3 100644 --- a/R/tar-terra-rast.R +++ b/R/tar-terra-rast.R @@ -89,7 +89,7 @@ tar_terra_rast <- function(name, # ensure that user-passed `resources` doesn't include `custom_format` check_user_resources(resources) - + if (preserve_metadata == "gdalraster_sozip") { check_pkg_installed("gdalraster") } @@ -189,18 +189,19 @@ tar_rast_write <- function(filetype, gdal, preserve_metadata) { gdalraster_sozip = function(object, path) { tmp <- withr::local_tempdir() - - dir.create(file.path(tmp, dirname(path)), recursive = TRUE) + dirpath <- file.path(tmp, dirname(path)) + tmppath <- file.path(tmp, path) + dir.create(dirpath, recursive = TRUE) terra::writeRaster( object, - file.path(tmp, path), + tmppath, filetype = filetype, overwrite = TRUE, gdal = gdal ) - raster_files <- list.files(file.path(tmp, dirname(path)), full.names = TRUE) + raster_files <- list.files(dirpath, full.names = TRUE) # create seek-optimized zip file using gdalraster gdalraster::addFilesInZip( @@ -216,7 +217,7 @@ tar_rast_write <- function(filetype, gdal, preserve_metadata) { # TODO: allow user control of number of threads? # how does num_threads interact multiple workers etc.? - unlink(file.path(tmp, path)) + unlink(tmppath) }, drop = function(object, path) { terra::writeRaster( From 8c4afd7d82389342108340886e3ba3edba2a1148 Mon Sep 17 00:00:00 2001 From: Andrew Gene Brown Date: Mon, 2 Dec 2024 07:39:41 -0800 Subject: [PATCH 08/12] temp paths for zip method --- R/tar-terra-rast.R | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/R/tar-terra-rast.R b/R/tar-terra-rast.R index 3ffd4a3..22d7182 100644 --- a/R/tar-terra-rast.R +++ b/R/tar-terra-rast.R @@ -164,7 +164,9 @@ tar_rast_write <- function(filetype, gdal, preserve_metadata) { zip = function(object, path) { #write the raster in a fresh local tempdir() that disappears when function is done tmp <- withr::local_tempdir() - dir.create(file.path(tmp, dirname(path)), recursive = TRUE) + dirpath <- file.path(tmp, dirname(path)) + tmppath <- file.path(tmp, basename(path)) + dir.create(dirpath, recursive = TRUE) terra::writeRaster( object, file.path(tmp, path), @@ -173,9 +175,9 @@ tar_rast_write <- function(filetype, gdal, preserve_metadata) { gdal = gdal ) #package files into a zip file using `zip::zip()` - raster_files <- list.files(file.path(tmp, dirname(path)), full.names = TRUE) + raster_files <- list.files(dirpath, full.names = TRUE) zip::zip( - file.path(tmp, basename(path)), + tmppath, files = raster_files, compression_level = 1, mode = "cherry-pick", @@ -183,8 +185,8 @@ tar_rast_write <- function(filetype, gdal, preserve_metadata) { ) # move the zip file to the expected place - file.copy(file.path(tmp, basename(path)), path) - unlink(file.path(tmp, basename(path))) + file.copy(tmppath, path) + unlink(tmppath) }, gdalraster_sozip = function(object, path) { From ee7b22a23917ac9d484f8da99337d7d724b05621 Mon Sep 17 00:00:00 2001 From: Andrew Gene Brown Date: Mon, 2 Dec 2024 07:45:03 -0800 Subject: [PATCH 09/12] whitespace --- R/tar-terra-rast.R | 167 ++++++++++++++++++++++----------------------- 1 file changed, 83 insertions(+), 84 deletions(-) diff --git a/R/tar-terra-rast.R b/R/tar-terra-rast.R index 22d7182..10005cb 100644 --- a/R/tar-terra-rast.R +++ b/R/tar-terra-rast.R @@ -144,91 +144,90 @@ tar_terra_rast <- function(name, } tar_rast_read <- function(preserve_metadata) { - switch( - preserve_metadata, - zip = function(path) { - tmp <- withr::local_tempdir() - zip::unzip(zipfile = path, exdir = tmp) - terra::rast(file.path(tmp, basename(path))) - }, - gdalraster_sozip = function(path) { - terra::rast(paste0("/vsizip/{", path, "}/"), basename(path)) - }, - drop = function(path) terra::rast(path) - ) + switch( + preserve_metadata, + zip = function(path) { + tmp <- withr::local_tempdir() + zip::unzip(zipfile = path, exdir = tmp) + terra::rast(file.path(tmp, basename(path))) + }, + gdalraster_sozip = function(path) { + terra::rast(paste0("/vsizip/{", path, "}/"), basename(path)) + }, + drop = function(path) terra::rast(path) + ) } tar_rast_write <- function(filetype, gdal, preserve_metadata) { - switch( - preserve_metadata, - zip = function(object, path) { - #write the raster in a fresh local tempdir() that disappears when function is done - tmp <- withr::local_tempdir() - dirpath <- file.path(tmp, dirname(path)) - tmppath <- file.path(tmp, basename(path)) - dir.create(dirpath, recursive = TRUE) - terra::writeRaster( - object, - file.path(tmp, path), - filetype = filetype, - overwrite = TRUE, - gdal = gdal - ) - #package files into a zip file using `zip::zip()` - raster_files <- list.files(dirpath, full.names = TRUE) - zip::zip( - tmppath, - files = raster_files, - compression_level = 1, - mode = "cherry-pick", - root = dirname(raster_files)[1] - ) - - # move the zip file to the expected place - file.copy(tmppath, path) - unlink(tmppath) - }, - gdalraster_sozip = function(object, path) { - - tmp <- withr::local_tempdir() - dirpath <- file.path(tmp, dirname(path)) - tmppath <- file.path(tmp, path) - dir.create(dirpath, recursive = TRUE) - - terra::writeRaster( - object, - tmppath, - filetype = filetype, - overwrite = TRUE, - gdal = gdal - ) - - raster_files <- list.files(dirpath, full.names = TRUE) - - # create seek-optimized zip file using gdalraster - gdalraster::addFilesInZip( - path, - raster_files, - full_paths = FALSE, - overwrite = TRUE, - sozip_enabled = "YES", - num_threads = 1, - quiet = TRUE - ) - # always create sozip regardless of file size (sozip_enabled = "YES") - # TODO: allow user control of number of threads? - # how does num_threads interact multiple workers etc.? - - unlink(tmppath) - }, - drop = function(object, path) { - terra::writeRaster( - object, - path, - filetype = filetype, - overwrite = TRUE, - gdal = gdal - ) - } - ) + switch( + preserve_metadata, + zip = function(object, path) { + #write the raster in a fresh local tempdir() that disappears when function is done + tmp <- withr::local_tempdir() + dirpath <- file.path(tmp, dirname(path)) + tmppath <- file.path(tmp, basename(path)) + dir.create(dirpath, recursive = TRUE) + terra::writeRaster( + object, + file.path(tmp, path), + filetype = filetype, + overwrite = TRUE, + gdal = gdal + ) + #package files into a zip file using `zip::zip()` + raster_files <- list.files(dirpath, full.names = TRUE) + zip::zip( + tmppath, + files = raster_files, + compression_level = 1, + mode = "cherry-pick", + root = dirname(raster_files)[1] + ) + + # move the zip file to the expected place + file.copy(tmppath, path) + unlink(tmppath) + }, + gdalraster_sozip = function(object, path) { + + tmp <- withr::local_tempdir() + dirpath <- file.path(tmp, dirname(path)) + tmppath <- file.path(tmp, path) + dir.create(dirpath, recursive = TRUE) + + terra::writeRaster( + object, + tmppath, + filetype = filetype, + overwrite = TRUE, + gdal = gdal + ) + raster_files <- list.files(dirpath, full.names = TRUE) + + # create seek-optimized zip file using gdalraster + gdalraster::addFilesInZip( + path, + raster_files, + full_paths = FALSE, + overwrite = TRUE, + sozip_enabled = "YES", + num_threads = 1, + quiet = TRUE + ) + # always create sozip regardless of file size (sozip_enabled = "YES") + # TODO: allow user control of number of threads? + # how does num_threads interact multiple workers etc.? + + unlink(tmppath) + }, + drop = function(object, path) { + terra::writeRaster( + object, + path, + filetype = filetype, + overwrite = TRUE, + gdal = gdal + ) + } + ) } From 9033bce510d9b3b48a0dcdd6b8cc106d6b7f2d63 Mon Sep 17 00:00:00 2001 From: Andrew Gene Brown Date: Mon, 2 Dec 2024 07:51:45 -0800 Subject: [PATCH 10/12] add test of gdalraster_sozip metadata --- tests/testthat/test-tar-terra-rast.R | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/testthat/test-tar-terra-rast.R b/tests/testthat/test-tar-terra-rast.R index 2acdef3..48ad291 100644 --- a/tests/testthat/test-tar-terra-rast.R +++ b/tests/testthat/test-tar-terra-rast.R @@ -151,3 +151,28 @@ tar_test("metadata is maintained", { expect_equal(terra::units(x), rep("m", 3)) expect_equal(terra::time(x), as.Date("2024-10-01") + c(0, 1, 2)) }) + +tar_test("metadata is maintained (gdalraster SOZip)", { + skip_if_not_installed("gdalraster") + tar_script({ + library(targets) + library(geotargets) + library(terra) + geotargets_option_set(terra_preserve_metadata = "gdalraster_sozip") + make_rast <- function() { + f <- system.file("ex/elev.tif", package = "terra") + r <- terra::rast(f) + r <- c(r, r + 10, r / 2) + terra::units(r) <- rep("m", 3) + terra::time(r) <- as.Date("2024-10-01") + c(0, 1, 2) + r + } + list( + tar_terra_rast(r, make_rast()) + ) + }) + tar_make() + x <- tar_read(r) + expect_equal(terra::units(x), rep("m", 3)) + expect_equal(terra::time(x), as.Date("2024-10-01") + c(0, 1, 2)) +}) From 02ca360a7e1515080337a319a8ef4cfdc45afe2a Mon Sep 17 00:00:00 2001 From: Andrew Gene Brown Date: Mon, 2 Dec 2024 07:54:22 -0800 Subject: [PATCH 11/12] fix broken read path! --- R/tar-terra-rast.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/tar-terra-rast.R b/R/tar-terra-rast.R index 10005cb..0f9a7df 100644 --- a/R/tar-terra-rast.R +++ b/R/tar-terra-rast.R @@ -152,7 +152,7 @@ tar_rast_read <- function(preserve_metadata) { terra::rast(file.path(tmp, basename(path))) }, gdalraster_sozip = function(path) { - terra::rast(paste0("/vsizip/{", path, "}/"), basename(path)) + terra::rast(paste0("/vsizip/{", path, "}/", basename(path))) }, drop = function(path) terra::rast(path) ) From 04015c6067aca0890f38f45fac087a7b5c6d5ad6 Mon Sep 17 00:00:00 2001 From: Andrew Gene Brown Date: Wed, 22 Jan 2025 23:39:23 -0800 Subject: [PATCH 12/12] improve test - use driver that stores metadata externally - include dataset and band user tags --- tests/testthat/test-tar-terra-rast.R | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/testthat/test-tar-terra-rast.R b/tests/testthat/test-tar-terra-rast.R index 48ad291..2dc1377 100644 --- a/tests/testthat/test-tar-terra-rast.R +++ b/tests/testthat/test-tar-terra-rast.R @@ -165,14 +165,18 @@ tar_test("metadata is maintained (gdalraster SOZip)", { r <- c(r, r + 10, r / 2) terra::units(r) <- rep("m", 3) terra::time(r) <- as.Date("2024-10-01") + c(0, 1, 2) + terra::metags(r) <- "FOO=BAR" + terra::metags(r, layer = 1) <- "asdf=hjkl" r } list( - tar_terra_rast(r, make_rast()) + tar_terra_rast(r, make_rast(), filetype = "HFA") ) }) tar_make() x <- tar_read(r) expect_equal(terra::units(x), rep("m", 3)) expect_equal(terra::time(x), as.Date("2024-10-01") + c(0, 1, 2)) + expect_equal(terra::metags(x), c(FOO = "BAR")) + expect_equal(terra::metags(x, 1), data.frame(layer = 1, name = "asdf", value = "hjkl")) })