RC v. 1.4.1 (#391)

ranghetti · Mar 2, 2021 · 17ccb69 · 17ccb69
1 parent ef9aeba
commit 17ccb69
Show file tree

Hide file tree

Showing 110 changed files with 1,126 additions and 803 deletions.
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,7 +1,7 @@
 Package: sen2r
 Type: Package
 Title: Find, Download and Process Sentinel-2 Data
-Version: 1.4.0.9003
+Version: 1.4.1
 Authors@R: c(person("Luigi", "Ranghetti",
                     email = "[email protected]",
                     role = c("aut", "cre"),

diff --git a/NEWS.md b/NEWS.md
@@ -1,3 +1,18 @@
+# Version 1.4.1
+
+## Minor changes
+- Prevent Copernicus mismatches between API Hub and `dhus` (#381)..
+- Rescale resolutions lower than required in `s2_translate()` (#368).
+- Avoid using progress bars in non-interactive sessions.
+- Manage error 429 for `dhus` (resend APIs more times in case of too many requests).
+
+## Bug fixes
+- Partially fix #368 (determine the output projection before processing).
+- Fix #371.
+- Fix #383.
+- Fix after changes in units 0.7-0 (#390).
+
+
 # Version 1.4.0
 
 ## Major changes

diff --git a/R/calcindex.R b/R/calcindex.R
@@ -78,7 +78,7 @@ calcindex_raster <- function(
   } else {
     bs <- blockSize(out, minrows = minrows)
   }
-  if (inherits(stdout(), "terminal")) {
+  if (all(inherits(stdout(), "terminal"), interactive())) {
     pb <- txtProgressBar(0, bs$n, style = 3)
   }
   for (i in seq_len(bs$n)) {
@@ -94,11 +94,11 @@ calcindex_raster <- function(
     }
     # m <- getValues(y, row = bs$row[i], nrows = bs$nrows[i])
     out <- writeValues(out, v_out, bs$row[i])
-    if (inherits(stdout(), "terminal")) {
+    if (all(inherits(stdout(), "terminal"), interactive())) {
       setTxtProgressBar(pb, i)
     }
   }
-  if (inherits(stdout(), "terminal")) {
+  if (all(inherits(stdout(), "terminal"), interactive())) {
     message("")
   }
   out <- writeStop(out)

diff --git a/R/check_sen2r_deps.R b/R/check_sen2r_deps.R
@@ -418,7 +418,7 @@ check_sen2r_deps <- function() {
         if (Sys.info()["machine"]=="x86-64") {"_64"},".exe"
       )
       osgeo4w_path <- tempfile(pattern="dir",fileext = ".exe")
-      out_bar <- if (inherits(stdout(), "terminal")) {
+      out_bar <- if (all(inherits(stdout(), "terminal"), interactive())) {
         NULL
       } else {
         file(out_bar_path <- tempfile(), open = "a")

diff --git a/R/gdal_warp.R b/R/gdal_warp.R
@@ -270,7 +270,7 @@ gdal_warp <- function(srcfiles,
     # Check that the polygon is not empty
     if (requireNamespace("units", quietly = TRUE)) {
       if (length(grep("POLYGON",st_geometry_type(mask)))>=1 &
-          sum(st_area(st_geometry(mask))) <= 0*units::ud_units$m^2) {
+          sum(st_area(st_geometry(mask))) <= 0*units::as_units("m^2")) {
         print_message(
           type = "error",
           "The polygon provided as mask cannot be empty."

diff --git a/R/install_aria2.R b/R/install_aria2.R
@@ -57,7 +57,7 @@ install_aria2 <- function(aria2_dir, force = FALSE) {
   aria2_zip <- normalize_path(file.path(aria2_dir,"aria2.zip"), mustWork=FALSE)
   aria2_path <- normalize_path(file.path(aria2_dir,"aria2c.exe"), mustWork=FALSE)
 
-  out_bar <- if (inherits(stdout(), "terminal")) {
+  out_bar <- if (all(inherits(stdout(), "terminal"), interactive())) {
     NULL
   } else {
     file(out_bar_path <- tempfile(), open = "a")

diff --git a/R/install_sen2cor.R b/R/install_sen2cor.R
@@ -126,7 +126,7 @@ install_sen2cor <- function(
   sen2cor_installer <- file.path(sen2cor_dir, basename(sen2cor_url))
 
   # download, extract and delete archive
-  out_bar <- if (inherits(stdout(), "terminal")) {
+  out_bar <- if (all(inherits(stdout(), "terminal"), interactive())) {
     NULL
   } else {
     file(out_bar_path <- tempfile(), open = "a")

diff --git a/R/s2_calcindices.R b/R/s2_calcindices.R
@@ -86,7 +86,6 @@
 #' @importFrom jsonlite fromJSON
 #' @import data.table
 #' @importFrom raster brick
-#' @importFrom utils txtProgressBar setTxtProgressBar
 #' @author Luigi Ranghetti, phD (2020) \email{luigi@@ranghetti.info}
 #' @references L. Ranghetti, M. Boschetti, F. Nutini, L. Busetto (2020).
 #'  "sen2r": An R toolbox for automatically downloading and preprocessing 

diff --git a/R/s2_download.R b/R/s2_download.R
@@ -301,19 +301,25 @@ s2_download <- function(
 
       if (downloader %in% c("builtin", "wget")) { # wget left for compatibility
 
-        out_bar <- if (inherits(stdout(), "terminal")) {
+        out_bar <- if (all(inherits(stdout(), "terminal"), interactive())) {
           NULL
         } else {
           file(out_bar_path <- tempfile(), open = "a")
         }
-        download <- RETRY(
-          verb = "GET",
-          url = as.character(link),
-          config = authenticate(creds[1], creds[2]),
-          times = 5, pause_cap = 8,
-          progress(con = if (length(out_bar) > 0) {out_bar} else {stdout()}),
-          write_disk(zip_path, overwrite = TRUE)
-        )
+
+        times_429 <- 10 # if 429 "too many requests", retry up to 10 times
+        while (times_429 > 0) {
+          download <- RETRY(
+            verb = "GET",
+            url = as.character(link),
+            config = authenticate(creds[1], creds[2]),
+            times = 5, pause_cap = 8,
+            progress(con = if (length(out_bar) > 0) {out_bar} else {stdout()}),
+            write_disk(zip_path, overwrite = TRUE)
+          )
+          times_429 <-if (download$status_code != 429) {0} else {times_429 - 1}
+        }
+
         if (length(out_bar) > 0) {
           close(out_bar)
           invisible(file.remove(out_bar_path))

diff --git a/R/s2_gui.R b/R/s2_gui.R
@@ -3162,7 +3162,7 @@ s2_gui <- function(param_list = NULL,
           "In this option:<ul>",
           "<li>set <strong>Yes</strong> to perform the topographic correction:",
           "DEM is searched in a default directory and downloaded from",
-          "<a href='http://srtm.csi.cgiar.org/' target='_blank'>CGIAR-CSI",
+          "<a href='https://srtm.csi.cgiar.org/' target='_blank'>CGIAR-CSI",
           "SRTM</a> if not found locally;</li>",
           "<li>set <strong>No</strong> to avoid applying a topographic",
           "correction;</li>",

diff --git a/R/s2_list.R b/R/s2_list.R
@@ -416,11 +416,16 @@ s2_list <- function(spatial_extent = NULL,
       query_string <- gsub("\\[", "%5b",query_string)
       query_string <- gsub("\\]", "%5d",query_string)
 
-      out_query <- RETRY(
-        verb = "GET",
-        url = query_string,
-        config = authenticate(creds[1,1], creds[1,2])
-      )
+      times_429 <- 10 # if 429 "too many requests", retry up to 10 times
+      while (times_429 > 0) {
+        out_query <- RETRY(
+          verb = "GET",
+          url = query_string,
+          config = authenticate(creds[1,1], creds[1,2])
+        )
+        times_429 <-if (out_query$status_code != 429) {0} else {times_429 - 1}
+      }
+
       out_xml <- content(out_query, as = "parsed", encoding = "UTF-8")
       out_xml_list <- xmlRoot(htmlTreeParse(out_xml, useInternalNodes = TRUE))
       out_xml_list <- out_xml_list[["body"]][["feed"]]

diff --git a/R/s2_mask.R b/R/s2_mask.R
@@ -621,7 +621,7 @@ s2_mask <- function(infiles,
               )
               #4 bytes per cell, nb + 1 bands (brick + mask), * 2 to account for a copy
               bs <- blockSize(out, minblocks = 8)
-              if (inherits(stdout(), "terminal")) {
+              if (all(inherits(stdout(), "terminal"), interactive())) {
                 pb <- txtProgressBar(0, bs$n, style = 3)
               }
               for (j in seq_len(bs$n)) {
@@ -633,11 +633,11 @@ s2_mask <- function(infiles,
 
                 out <- writeValues(out, v, bs$row[j])
                 gc()
-                if (inherits(stdout(), "terminal")) {
+                if (all(inherits(stdout(), "terminal"), interactive())) {
                   setTxtProgressBar(pb, j)
                 }
               }
-              if (inherits(stdout(), "terminal")) {
+              if (all(inherits(stdout(), "terminal"), interactive())) {
                 message("")
               }
               out <- writeStop(out)

diff --git a/R/s2_order.R b/R/s2_order.R
@@ -223,12 +223,17 @@ s2_order <- function(
     if (i != to_order[1]) {
       Sys.sleep(delay)
     }
+
     # order products
-    make_order <- RETRY(
-      verb = "GET",
-      url = as.character(s2_prodlist[i]),
-      config = authenticate(creds[1], creds[2])
-    )
+    times_429 <- 10 # if 429 "too many requests", retry up to 10 times
+    while (times_429 > 0) {
+      make_order <- RETRY(
+        verb = "GET",
+        url = as.character(s2_prodlist[i]),
+        config = authenticate(creds[1], creds[2])
+      )
+      times_429 <-if (make_order$status_code != 429) {0} else {times_429 - 1}
+    }
 
     # check if the order was successful
     if (inherits(make_order, "response")) {

diff --git a/R/s2_tiles.R b/R/s2_tiles.R
@@ -63,7 +63,7 @@ s2_tiles <- function() {
     #   file.path(dirname(attr(load_binpaths(), "path")), "s2_tiles.rds")
     # )
 
-    out_bar <- if (inherits(stdout(), "terminal")) {
+    out_bar <- if (all(inherits(stdout(), "terminal"), interactive())) {
       NULL
     } else {
       file(out_bar_path <- tempfile(), open = "a")

diff --git a/R/s2_translate.R b/R/s2_translate.R
@@ -295,22 +295,22 @@ s2_translate <- function(infile,
           # define the steps to perform:
           # 1. create a multiband stack
           do_step1 <- length(jp2_selbands)>1
-          # 2: rescale in case of wrong native resolution
+          # 2: rescale L2A in case of wrong native resolution
           # (i.e. SAFE not containing 10m bands with res = "10m")
           res_out <- if (all(
             prod_type %in% c("SCL","CLD","SNW"),
             res[1] == "10m"
           )) {res[2]} else {res[1]}
           res_ratio <- min(as.integer(substr(jp2df_selbands$res,1,2))) / 
             as.integer(substr(res_out,1,2))
-          do_step2 <- res_ratio > 1 # it could make sense define for != 1
+          do_step2 <- infile_meta$level == "2A" && res_ratio > 1
           # 3. mask nodata values if the tile does not completely cover the orbit
           infile_footprint <- st_transform(
             st_as_sfc(infile_meta$footprint, crs = 4326), 
             st_crs2(sel_utmzone)
           )
           infile_area <- sum(st_area(infile_footprint))
-          do_step3 <- infile_area < 109e3^2*units::ud_units$m^2
+          do_step3 <- infile_area < 109e3^2*units::as_units("m^2")
           # 4. convert the final VRT to a physical format
           do_step4 <- format != "VRT"
 

diff --git a/R/sen2cor.R b/R/sen2cor.R
@@ -61,7 +61,7 @@
 #'      with the function `read_gipp(c("DEM_Directory", "DEM_Reference"))`).
 #'      In case one or both these parameters were set to `"NONE"`, 
 #'      a subdirectory `"srtm90"` of the default sen2r directory is used as
-#'      DEM directory, and/or the [CGIAR SRTM 90m](http://srtm.csi.cgiar.org/) 
+#'      DEM directory, and/or the [CGIAR SRTM 90m](https://srtm.csi.cgiar.org/) 
 #'      is set as online source.
 #'      To set another directory or reference, use argument `gipp` in the form
 #'      `gipp = list(DEM_Directory = tempdir(), DEM_Reference ="another_reference", ...)`

diff --git a/cran-comments.md b/cran-comments.md
@@ -1,20 +1,27 @@
-# sen2r v. 1.4.0
+# sen2r v. 1.4.1
 
 ## Test environments
-* [local installation] Ubuntu 18.04, 64 bit, R 4.0.3
-* [local installation] Archlinux, 64 bit, R 4.0.3
+* [local installation] Ubuntu 18.04, 64 bit, R 4.0.4
+* [local installation] Archlinux, 64 bit, R 4.0.4
 * [local installation] Windows 10, 64 bit, R 4.0.3
-* [win-builder] R unstable, 4.0.3 and 3.6.3 (devel, release and oldrelease)
+* [win-builder] R unstable, 4.0.4 and 3.6.3 (devel, release and oldrelease)
 
 ## R CMD check results
 There were no ERRORs, WARNINGs or NOTEs.
 
 
 ## CRAN review
-> The auto-check found additional issues for the *last* version released on CRAN:
-  M1mac <https://www.stats.ox.ac.uk/pub/bdr/M1mac/sen2r.out>
-CRAN incoming checks do not test for these additional issues and you will need
-an appropriately instrumented build of R to reproduce these.
-Hence please reply-all and explain: Have these been fixed?
+> Dear maintainer,
+Please see the problems shown on
+<https://cran.r-project.org/web/checks/check_results_sen2r.html>.
+Please correct before 2021-03-11 to safely retain your package on CRAN.
 
-All the errors found in M1mac were fixed (see the NEWS.md file for details).
+All the errors were fixed (they were due to #390)
+(see commit 82754724fd3259b12ea1aee57a5242cfcec8438d).
+
+> 'Writing R Extensions' asked you not to use progress bars in
+non-interactive sessions and doing so makes your erroneous output
+necessarily hard to read.
+
+All progress bars were put within `if (interactive())` checks 
+(see commit c4f36378859d584d9c617ccdd3938d4086a05816).
diff --git a/docs/404.html b/docs/404.html