RC v. 1.3.1 (#273)

## Major changes - Documentation was improved with three new vignettes about GUI usage, command line usage and output structure. ## Bug fixes - Restore checking environmental variable in `init_python()` (this urgent bug fixing was the reason of the urgent release of the current version). - Fix building offline footprint. - Fix the Dockerfile. ## Changes in default argument values - Accept both `sen2r(..., rm_safe = "yes"`) and `"all"`. - In the GUI, the maximum allowed cloud cover was set to 100% accordingly to the `sen2r()` defaults. ## Minor changes - Add check on `max_mask` - `mask_type` coherence (if `max_mask < 100` and `mask_type` is not specified, a warning is returned). - Suppress some unuseful warnings. - Switch examples on 2019 dates (previously examples were based on 2017 images, which were partially moved on LTA). - Do not print progress bars in logs.
ranghetti · Feb 7, 2020 · 668d351 · 668d351
1 parent 030e9bf
commit 668d351
Show file tree

Hide file tree

Showing 174 changed files with 3,198 additions and 696 deletions.
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,7 +1,7 @@
 Package: sen2r
 Type: Package
 Title: Find, Download and Process Sentinel-2 Data
-Version: 1.3.0
+Version: 1.3.1
 Authors@R: c(person("Luigi", "Ranghetti",
                     email = "[email protected]",
                     role = c("aut", "cre"),
@@ -23,7 +23,6 @@ BugReports: https://github.com/ranghetti/sen2r/issues
 Depends: R (>= 3.5.0)
 Imports:
   methods,
-  rgdal,
   sf,
   stars,
   data.table,

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM rocker/geospatial:3.5.1
+FROM rocker/geospatial:latest
 
 LABEL maintainer="Luigi Ranghetti <[email protected]>"
 
@@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y \
         python-gdal \
         aria2 \
         libpython-dev \
-        libv8-3.14-dev && \
+        libnode-dev && \
     apt-get autoremove -y && \
     rm -rf /var/lib/apt/lists/*
 

diff --git a/NAMESPACE b/NAMESPACE
@@ -276,4 +276,6 @@ importFrom(utils,getFromNamespace)
 importFrom(utils,head)
 importFrom(utils,packageVersion)
 importFrom(utils,read.table)
+importFrom(utils,setTxtProgressBar)
+importFrom(utils,txtProgressBar)
 importFrom(utils,unzip)
diff --git a/NEWS.md b/NEWS.md
@@ -1,3 +1,24 @@
+# Version 1.3.1
+
+## Major changes
+- Documentation was improved with three new vignettes about GUI usage, command line usage and output structure.
+
+## Bug fixes
+- Restore checking environmental variable in `init_python()` (this urgent bug fixing was the reason of the urgent release of the current version).
+- Fix building offline footprint.
+- Fix the Dockerfile.
+
+## Changes in default argument values
+- Accept both `sen2r(..., rm_safe = "yes"`) and `"all"`.
+- In the GUI, the maximum allowed cloud cover was set to 100% accordingly to the `sen2r()` defaults.
+
+## Minor changes
+- Add check on `max_mask` - `mask_type` coherence (if `max_mask < 100` and `mask_type` is not specified, a warning is returned).
+- Suppress some useless warnings.
+- Switch examples on 2019 dates (previously examples were based on 2017 images, which were partially moved on LTA).
+- Do not print progress bars in logs.
+
+
 # Version 1.3.0
 
 ## Major changes

diff --git a/R/build_example_param_file.R b/R/build_example_param_file.R
@@ -56,7 +56,7 @@ build_example_param_file <- function(
         "step_atmcorr" = "l2a",
         "sen2cor_use_dem" = NA,
         "sen2cor_gipp" = NA,
-        "timewindow" = c("2017-07-03", "2017-07-03"),
+        "timewindow" = c("2019-07-23", "2019-07-23"),
         "timeperiod" = "full",
         "extent" = system.file("extdata/vector/barbellino.geojson", package = "sen2r"),
         "s2tiles_selected" = NA,
@@ -70,7 +70,7 @@ build_example_param_file <- function(
         ),
         "index_source" = "BOA",
         "mask_type" = NA,
-        "max_mask" = 80,
+        "max_mask" = 100,
         "mask_smooth" = 0,
         "mask_buffer" = 0,
         "clip_on_extent" = TRUE,

diff --git a/R/check_gdal.R b/R/check_gdal.R
@@ -340,7 +340,8 @@ check_gdal <- function(abort = TRUE, gdal_path = NULL, force = FALSE, full_scan
 
   print_message(
     type="message",
-    "GDAL version in use: ", as.character(gdal_version))
+    "GDAL version in use: ", as.character(gdal_version)
+  )
   return(invisible(TRUE))
 
 }
diff --git a/R/check_param_list.R b/R/check_param_list.R
@@ -66,7 +66,7 @@ check_param_list <- function(pm, type = "string", check_paths = FALSE, correct =
   for (sel_par in names(pm_def)) {
     if (length(nn(pm[[sel_par]])) == 0) {
       print_message(
-        type = "warning",
+        type = if (type == "error") {"warning"} else {type},
         paste0("Parameter \"",sel_par,"\" was not specified; ",
                "setting it to the default ('",pm_def[[sel_par]],"').")
       )
@@ -154,10 +154,11 @@ check_param_list <- function(pm, type = "string", check_paths = FALSE, correct =
     pm$rm_safe <- "yes"
   } else if (pm$rm_safe == FALSE) {
     pm$rm_safe <- "no"
-  } else if (!pm$rm_safe %in% c("yes", "no", "l1c")) {
+  } else if (!pm$rm_safe %in% c("yes", "all", "no", "l1c")) {
     print_message(
       type = type,
-      "Parameter \"online\" must be one among 'yes', 'no' and 'l1c' (setting to the default)."
+      "Parameter \"rm_safe\" must be one among 'yes' (or 'all'), ",
+      "'no' and 'l1c' (setting to the default)."
     )
     pm$rm_safe <- pm_def$rm_safe
   }
@@ -177,14 +178,14 @@ check_param_list <- function(pm, type = "string", check_paths = FALSE, correct =
   }
   if (pm$max_cloud_safe < 0) {
     print_message(
-      type = "warning",
+      type = if (type == "error") {"warning"} else {type},
       "Minimum allowed cloud cover value is 0; ",
       "setting parameter \"max_cloud_safe\" to 0."
     )
     pm$max_cloud_safe <- 0
   } else if (pm$max_cloud_safe > 100) {
     print_message(
-      type = "warning",
+      type = if (type == "error") {"warning"} else {type},
       "Maximum allowed cloud cover value is 100; ",
       "setting parameter \"max_cloud_safe\" to 100."
     )
@@ -432,19 +433,26 @@ check_param_list <- function(pm, type = "string", check_paths = FALSE, correct =
   }
   if (pm$max_mask < 0) {
     print_message(
-      type = "warning",
+      type = if (type == "error") {"warning"} else {type},
       "Minimum allowed cloud cover value is 0; ",
       "setting parameter \"max_mask\" to 0."
     )
     pm$max_mask <- 0
   } else if (pm$max_mask > 100) {
     print_message(
-      type = "warning",
+      type = if (type == "error") {"warning"} else {type},
       "Maximum allowed cloud cover value is 100; ",
       "setting parameter \"max_mask\" to 100."
     )
     pm$max_mask <- 0
   }
+  if (all(pm$max_mask < 100, is.na(pm$mask_type))) {
+    print_message(
+      type = if (type == "error") {"warning"} else {type},
+      "\"max_mask\" was set to ",pm$max_mask,", but no masks were defined: ",
+      "set argument \"mask_type\" properly in order to use a cloud mask."
+    )
+  }
 
 
   # -- mask_smooth --
@@ -551,7 +559,7 @@ check_param_list <- function(pm, type = "string", check_paths = FALSE, correct =
   # -- res_s2 --
   if ((!anyNA(pm$res) & !is.null(pm$res)) & (!anyNA(pm$res_s2) & !is.null(pm$res_s2))) {
     print_message(
-      type = "warning",
+      type = if (type == "error") {"warning"} else {type},
       "Both native and custom resolution were provided; ",
       "only custom one (\"res\") will be used."
     )
@@ -575,7 +583,7 @@ check_param_list <- function(pm, type = "string", check_paths = FALSE, correct =
   # -- unit --
   if (pm$unit != "Meter") {
     print_message(
-      type = "warning",
+      type = if (type == "error") {"warning"} else {type},
       "Only \"unit\" == 'meter' is accepted."
     )
     pm$unit <- "Meter"
@@ -718,7 +726,7 @@ check_param_list <- function(pm, type = "string", check_paths = FALSE, correct =
   # -- step_atmcorr --
   if (pm$step_atmcorr == "no") {
     print_message(
-      type = "warning",
+      type = if (type == "error") {"warning"} else {type},
       "Value \"no\" for parameter \"step_atmcorr\" is deprecated ",
       "(\"l2a\" will be used)."
     )

diff --git a/R/gdal_abs2rel.R b/R/gdal_abs2rel.R
@@ -18,7 +18,7 @@
 #' @examples
 #' # Load a VRT containing a relative path
 #' ex_vrt <- system.file(
-#'   "extdata/out/S2A2A_20170703_022_Barbellino_RGB432B_10.vrt", 
+#'   "extdata/out/S2A2A_20190723_022_Barbellino_RGB432B_10.vrt", 
 #'   package = "sen2r"
 #' )
 #' abs_vrt <- tempfile(fileext = "_abs.vrt")

diff --git a/R/gdal_warp.R b/R/gdal_warp.R
@@ -62,11 +62,11 @@
 #' \donttest{
 #' #' # Define file names
 #' ex_sel <- system.file(
-#'   "extdata/out/S2A2A_20170703_022_Barbellino_RGB432B_10.tif",
+#'   "extdata/out/S2A2A_20190723_022_Barbellino_RGB432B_10.tif",
 #'   package = "sen2r"
 #' )
 #' ex_ref <- system.file(
-#'   "extdata/out/S2A2A_20170703_022_Barbellino_SCL_10.tif",
+#'   "extdata/out/S2A2A_20190723_022_Barbellino_SCL_10.tif",
 #'   package = "sen2r"
 #' )
 #' crop_poly <- system.file("extdata/vector/dam.geojson", package = "sen2r")
@@ -356,7 +356,7 @@ gdal_warp <- function(srcfiles,
         if (is.null(mask)) {
           # ref NULL & mask NULL: use bbox of srcfile, reprojected
           sel_te <- sel_src_bbox
-        } else if (class(mask)=="logical" && is.na(mask)) { # check if mask==NA
+        } else if (inherits(mask, "logical") && is.na(mask)) { # check if mask==NA
           # ref NULL & mask NA: the same (use bbox of srcfile, reprojected)
           sel_te <- sel_src_bbox
         } else {
@@ -383,7 +383,7 @@ gdal_warp <- function(srcfiles,
         if (is.null(mask)) {
           # ref provided & mask NULL: use bbox of ref
           sel_te <- ref_bbox
-        } else if (class(mask)=="logical" && is.na(mask)) {
+        } else if (inherits(mask, "logical") && is.na(mask)) {
           # ref provided & mask NA: use bbox of srcfile (reprojected and aligned to ref grid)
           if (sel_t_srs == sel_s_srs) {
             sel_te <- (sel_src_bbox - ref_ll) / sel_tr

diff --git a/R/gdalwarp_grid.R b/R/gdalwarp_grid.R
@@ -20,11 +20,11 @@
 #' \donttest{
 #' # Define file names
 #' ex_sel <- system.file(
-#'   "extdata/out/S2A2A_20170703_022_Barbellino_BOA_10.tif",
+#'   "extdata/out/S2A2A_20190723_022_Barbellino_BOA_10.tif",
 #'   package = "sen2r"
 #' )
 #' ex_ref <- system.file(
-#'   "extdata/out/S2A2A_20170703_022_Barbellino_SCL_10.tif",
+#'   "extdata/out/S2A2A_20190723_022_Barbellino_SCL_10.tif",
 #'   package = "sen2r"
 #' )
 #' ex_out <- tempfile(fileext = "_BOA_out.tif")

diff --git a/R/init_python.R b/R/init_python.R
@@ -1,7 +1,42 @@
 #' @title Initialise python
-#' @description Deprecated function.
+#' @description Internal function to set the environmental variables
+#'  required to run Python-based GDAL utilities on Windows.
 #' @return NULL (the function is called for its side effects)
+#' @author Luigi Ranghetti, phD (2020) \email{luigi@@ranghetti.info}
+#' @note License: GPL 3.0
 
 init_python <- function() {
-  stop("This function is deprecated.")
+
+  # Setting environmental variables is required only on Windows
+  if (Sys.info()["sysname"] != "Windows") {
+    return(invisible(NULL))
+  }
+
+  binpaths <- load_binpaths("gdal")
+  pythonhome_new <- list.files(
+    file.path(dirname(dirname(binpaths$gdalinfo)),"apps"), 
+    pattern="^Python", 
+    full.names=TRUE
+  )[1]
+  pythonhome_exi <- Sys.getenv("PYTHONHOME")
+  if (!normalize_path(pythonhome_exi) %in% normalize_path(pythonhome_new)) {
+    Sys.setenv(PYTHONHOME = pythonhome_new)
+    on.exit(Sys.setenv(PYTHONHOME = pythonhome_exi))
+  }
+  pythonpath_new <- list.files(pythonhome_new,"^[Ll]ib",full.names=TRUE)[1]
+  pythonpath_exi <- Sys.getenv("PYTHONPATH")
+  if (!normalize_path(pythonpath_exi) %in% normalize_path(pythonpath_new)) {
+    Sys.setenv(PYTHONPATH = pythonpath_new)
+    on.exit(Sys.setenv(PYTHONPATH = pythonpath_exi))
+  }
+  path_exi <- Sys.getenv("PATH")
+  if (!any(grepl(
+    normalize_path(pythonhome_new), 
+    normalize_path(unlist(strsplit(path_exi, ";")), mustWork = FALSE), 
+    fixed=TRUE
+  ))) {
+    Sys.setenv(PATH = paste0(pythonhome_new,";",Sys.getenv("PATH")))
+    on.exit(Sys.setenv(PATH = path_exi))
+  }
+
 }
diff --git a/R/load_binpaths.R b/R/load_binpaths.R
@@ -76,12 +76,6 @@ load_binpaths <- function(bins = NULL) {
     }
   }
 
-  # # Check python
-  # if ("python" %in% bins & is.null(binpaths$python)) {
-  #   init_python()
-  #   binpaths <- jsonlite::fromJSON(binpaths_file)
-  # }
-
   # Return the list
   attr(binpaths, "path") <- binpaths_file
   binpaths

diff --git a/R/print_message.R b/R/print_message.R
@@ -54,10 +54,16 @@ print_message <- function(
     initial <- paste0(initial, "[",strftime(Sys.time(), format=date_format),"] ")
   }
   if (all(is.logical(exdent), exdent)) {
-    exdent <- max(0, nchar(initial) - nchar(prefix))
+    exdent <- max(0, min(
+      nchar(initial) - nchar(prefix),
+      .1 * width
+    ))
   }
   if (all(is.logical(indent), indent)) {
-    indent <- max(0, nchar(prefix) - nchar(initial))
+    indent <- max(0, min(
+      nchar(prefix) - nchar(initial),
+      .1 * width
+    ))
   }
   message_string <- strwrap(
     unlist(strsplit(paste(c(...), collapse=sep),"\n")),

diff --git a/R/raster_metadata.R b/R/raster_metadata.R
@@ -20,7 +20,7 @@
 #' examplenames <- c(
 #'   system.file("tif/L7_ETMs.tif", package="stars"),
 #'   system.file("nc/bcsd_obs_1999.nc", package = "stars"),
-#'   system.file("extdata/out/S2A2A_20170703_022_Barbellino_BOA_10.tif",
+#'   system.file("extdata/out/S2A2A_20190723_022_Barbellino_BOA_10.tif",
 #'     package = "sen2r")
 #' )
 #'

diff --git a/R/s2_calcindices.R b/R/s2_calcindices.R
@@ -85,12 +85,13 @@
 #' @import data.table
 #' @importFrom raster blockSize brick getValues raster writeStart writeStop writeValues
 #' @importFrom stars read_stars write_stars
+#' @importFrom utils txtProgressBar setTxtProgressBar
 #' @author Luigi Ranghetti, phD (2020) \email{luigi@@ranghetti.info}
 #' @note License: GPL 3.0
 #' @examples
 #' # Define file names
 #' ex_in <- system.file(
-#'   "extdata/out/S2A2A_20170703_022_Barbellino_BOA_10.tif",
+#'   "extdata/out/S2A2A_20190723_022_Barbellino_BOA_10.tif",
 #'   package = "sen2r"
 #' )
 #'
@@ -169,8 +170,11 @@ s2_calcindices <- function(
     } else {
       bs <- blockSize(out, minrows = minrows)
     }
+    if (inherits(stdout(), "terminal")) {
+      pb <- txtProgressBar(0, bs$n, style = 3)
+    }
     for (i in seq_len(bs$n)) {
-      message("Processing chunk ", i, " of ", bs$n)
+      # message("Processing chunk ", i, " of ", bs$n)
       v <- getValues(x, row = bs$row[i], nrows = bs$nrows[i])
       if (grepl("^Float", dataType)) {
         if (!is.numeric(v)) {
@@ -180,10 +184,14 @@ s2_calcindices <- function(
       } else {
         v_out <- round(eval(parse(text = sel_formula)))
       }
-
-
       # m <- getValues(y, row = bs$row[i], nrows = bs$nrows[i])
       out <- writeValues(out, v_out, bs$row[i])
+      if (inherits(stdout(), "terminal")) {
+        setTxtProgressBar(pb, i)
+      }
+    }
+    if (inherits(stdout(), "terminal")) {
+      message("")
     }
     out <- writeStop(out)
     NULL
@@ -387,7 +395,14 @@ s2_calcindices <- function(
     # (this cycle is not parallelised)
     sel_outfiles <- character(0)
     for (j in seq_along(indices)) {
-      print_message(paste0("Computing index: ", indices[j]), type = "message")
+      print_message(
+        type = "message",
+        date = TRUE,
+        paste0(
+          "Computing index ", indices[j],
+          " on date ",sel_infile_meta$sensing_date,"..."
+        )
+      )
       # extract parameters
       sel_parameters <- parameters[[indices[j]]]
 
@@ -492,6 +507,7 @@ s2_calcindices <- function(
 
         # Launch the processing
         if (proc_mode == "gdal_calc") {
+          init_python()
           system(
             paste0(
               binpaths$gdal_calc," ",