Parallel support improvements. (#124)

Co-authored-by: Mark Keller <[email protected]>

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: pizzarr
 Type: Package
-Title: Slice into Zarr arrays in R
+Title: Slice into Zarr Arrays in R
 Version: 0.1.0
 Authors@R: c(
     person(

R/options.R

@@ -1,21 +1,29 @@
 # Adapted from https://github.com/IRkernel/IRkernel/blob/master/R/options.r
 
 #' pizzarr_option_defaults
+#' @description 
+#' * pizzarr.http_store_cache_time_seconds how long to cache web requests
+#' * pizzarr.parallel_backend "future", a cluster object, or an integer (if not on windows)
+#' * pizzarr.parallel_write_enabled logical, whether to use parallel backend for writing
+#' * pizzarr.progress_bar logical whether to use `pbapply` to emit a progress bar
 #' @export
 pizzarr_option_defaults <- list(
     pizzarr.http_store_cache_time_seconds = 3600,
-    pizzarr.parallel_read_enabled = FALSE,
-    pizzarr.parallel_write_enabled = FALSE
+    pizzarr.parallel_backend = NA,
+    pizzarr.parallel_write_enabled = FALSE,
+    pizzarr.progress_bar = FALSE
 )
 
 #' @keywords internal
 parse_parallel_option <- function(val) {
 
+  if(is.na(val)) return(val)
+
   if(inherits(val, "cluster")) {
     return(val)
   }
 
-  if(val == "future") {
+  if(!is.na(val) && val == "future") {
     return("future")
   }
 
@@ -34,8 +42,9 @@ parse_parallel_option <- function(val) {
 #' @keywords internal
 from_env <- list(
     PIZZARR_HTTP_STORE_CACHE_TIME_SECONDS = as.integer,
-    PIZZARR_PARALLEL_READ_ENABLED = parse_parallel_option,
-    PIZZARR_PARALLEL_WRITE_ENABLED = parse_parallel_option
+    PIZZARR_PARALLEL_BACKEND = parse_parallel_option,
+    PIZZARR_PARALLEL_WRITE_ENABLED = as.logical,
+    PIZZARR_PROGRESS_BAR = as.logical
 )
 
 # converts e.g. jupyter.log_level to JUPYTER_LOG_LEVEL

R/stores.R

@@ -348,12 +348,14 @@ MemoryStore <- R6::R6Class("MemoryStore",
 #' @description
 #' Store class that uses HTTP requests.
 #' Read-only. Depends on the `crul` package.
-#' @details For parallel operation, set the "pizzarr.parallel_read_enabled" option
+#' @details For parallel operation, set the "pizzarr.parallel_backend" option
 #' to one of:
 #' * `"future"` if a future plan has been set up
 #' * `integer` if you would like a one-time use cluster created per call
 #' * `cluster` object created with `parallel::make_cluster()` if you want to reuse a cluster
 #' 
+#' Set the option "pizzarr.progress_bar" to TRUE to get a progress bar for long running reads.
+#' 
 #' For more, see `vignette("parallel").`
 #' @rdname HttpStore
 #' @importFrom memoise memoise timeout
@@ -375,7 +377,11 @@ HttpStore <- R6::R6Class("HttpStore",
       key <- item_to_key(item)
       path <- paste(private$base_path, key, sep="/")
 
-      parallel_option <- getOption("pizzarr.parallel_read_enabled")
+      ret <- try_from_zmeta(key, self)
+
+      if(!is.null(ret)) return(ret)
+
+      parallel_option <- getOption("pizzarr.parallel_backend", NA)
       parallel_option <- parse_parallel_option(parallel_option)
       is_parallel <- is_truthy_parallel_option(parallel_option)
 

R/utils.R

@@ -374,12 +374,12 @@ item_to_key <- function(item) {
 
 #' @keywords internal
 is_truthy_parallel_option <- function(val) {
-  if(inherits(val, "cluster")) {
-    return(TRUE)
-  }
-  if(val == "future") {
-    return(TRUE)
-  }
+  if(is.na(val)) return(FALSE)
+
+  if(inherits(val, "cluster")) return(TRUE)
+
+  if(val == "future") return(TRUE)
+
   return(as.logical(as.integer(val)))
 }