v. 1.3.8 (#344)

## Minor changes - Add argument `service` to `s2_list()` and `s2_download()` for using `"dhus"` API service instead of default `"apihub"` (this could be useful in case of `apihub` downtimes). - Add experimental argument `kill_errored` to `sen2cor()` (see documentation). - Add argument `server` to `s2_list()` to be used for future implementations. - Replace internal URLs https://raw.githubusercontet.com/x/y with https://github.com/x/y/raw. ## Changes in default values - `build_example_param_file()` now generates more recent images, 2020-08-01 instead than 2019-07-23 (this because the previous ones required two SAFE which are now on LTA); the same was done for some tests. ## Bug fixes - Fix #338
ranghetti · Aug 27, 2020 · 56c9c08 · 56c9c08
1 parent b0d8d41
commit 56c9c08
Show file tree

Hide file tree

Showing 133 changed files with 1,022 additions and 598 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,6 +1,6 @@
 language: R
 sudo: required
-dist: bionic
+dist: xenial
 cache: packages
 
 addons:
@@ -27,7 +27,7 @@ addons:
 
 before_install:
   - Rscript -e 'install.packages("remotes")'
-  - Rscript -e 'remotes::install_github("r-spatial/lwgeom", ref = "467dc314a0bda011e806a8b56c8dcc153f63527c")'
+  - Rscript -e 'remotes::install_github("cran/XML")' # required for R 3.6.3
 
 r:
   - release

diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,7 +1,7 @@
 Package: sen2r
 Type: Package
 Title: Find, Download and Process Sentinel-2 Data
-Version: 1.3.7
+Version: 1.3.8
 Authors@R: c(person("Luigi", "Ranghetti",
                     email = "[email protected]",
                     role = c("aut", "cre"),

diff --git a/NEWS.md b/NEWS.md
@@ -1,3 +1,18 @@
+# Version 1.3.8
+
+## Minor changes
+- Add argument `service` to `s2_list()` and `s2_download()` for using `"dhus"` API service instead of default `"apihub"` (this could be useful in case of `apihub` downtimes).
+- Add experimental argument `kill_errored` to `sen2cor()` (see documentation).
+- Add argument `server` to `s2_list()` to be used for future implementations.
+- Replace internal URLs https://raw.githubusercontet.com/x/y with https://github.com/x/y/raw.
+
+## Changes in default values
+- `build_example_param_file()` now generates more recent images, 2020-08-01 instead than 2019-07-23 (this because the previous ones required two SAFE which are now on LTA); the same was done for some tests.
+
+## Bug fixes
+- Fix #338
+
+
 # Version 1.3.7
 
 ## Major changes
@@ -271,7 +286,7 @@ remotes::install_packages_github("ranghetti/sen2r")
 * Some new indices were added:
     - NDWI-NDWI2 (#184);
     - CRred-BDred-CRred2 (#168);
-    - Indices from [Sentinel-hub indices](https://www.sentinel-hub.com/develop/documentation/eo_products/Sentinel2EOproducts) were checked.
+    - Indices from [Sentinel-hub indices](https://www.sentinel-hub.com/develop/documentation/eo_products/Sentinel2EOproducts/) were checked.
 * The minimum convex hull of the extent is used instead of the bbox to search SAFE images.
 * Several "soft" error/warning messages were introduced.
 * ENVI headers now includes some information about band names (TOA/BOA) and SCL classes.
@@ -343,7 +358,7 @@ First stable release of package **sen2r**! See the announcement [here](https://l
 * `tiles_intersects()`: when an extent is loaded, now only required tiles are automatically used, instead than all the overlapping ones.
 
 ### Update dependencies
-* after [V8 major update](https://www.r-bloggers.com/a-major-upgrade-of-the-v8-package/), [Installation page](https://ranghetti.github.io/sen2r/articles/installation.html) was edited to refer to `v8` instead than to `v8-3.14`
+* after [V8 major update](https://www.r-bloggers.com/a-major-upgrade-of-the-v8-package/), [Installation page](http://sen2r.ranghetti.info/articles/installation.html) was edited to refer to `v8` instead than to `v8-3.14`
 
 ### Bug fixing:
 * Fix bug in SciHub login

diff --git a/R/build_example_param_file.R b/R/build_example_param_file.R
@@ -61,7 +61,7 @@ build_example_param_file <- function(
         "step_atmcorr" = "l2a",
         "sen2cor_use_dem" = NA,
         "sen2cor_gipp" = NA,
-        "timewindow" = c("2019-07-23", "2019-07-23"),
+        "timewindow" = c("2020-08-01", "2020-08-01"),
         "timeperiod" = "full",
         "extent" = system.file("extdata/vector/barbellino.geojson", package = "sen2r"),
         "s2tiles_selected" = NA,

diff --git a/R/check_gui_deps.R b/R/check_gui_deps.R
@@ -44,8 +44,8 @@ check_gui_deps <- function(abort = TRUE) {
       type = ifelse(abort, "error", "warning"),
       "Some missing packages are needed to run the GUI; ",
       "please install them with the command \n",
-      " > install.packages(\"",
-      paste(names(gui_deps_missing)[gui_deps_missing], collapse = "\", \""),"\")"
+      " > install.packages(c(\"",
+      paste(names(gui_deps_missing)[gui_deps_missing], collapse = "\", \""),"\"))"
     )
     invisible(TRUE)
   } else {

diff --git a/R/gdal_warp.R b/R/gdal_warp.R
@@ -319,9 +319,10 @@ gdal_warp <- function(srcfiles,
     # create mask_bbox if t_srs is specified;
     # otherwise, create each time within srcfile cycle
     if (!is.null(t_srs)) {
-      mask_bbox <- st_bbox(
-        st_transform(mask, t_srs),
-        matrix(nrow=2, ncol=2, dimnames=list(c("x","y"),c("min","max")))
+      mask_bbox <- matrix(
+        st_bbox(st_transform(mask, t_srs)),
+        nrow=2, ncol=2, 
+        dimnames=list(c("x","y"),c("min","max"))
       )
     }
   }

diff --git a/R/install_sen2cor.R b/R/install_sen2cor.R
@@ -1,6 +1,6 @@
 #' @title Download and install (or link) Sen2Cor
 #' @description [install_sen2cor()] downloads and installs a standalone version of
-#'  [Sen2Cor](http://step.esa.int/main/third-party-plugins-2/sen2cor).
+#'  [Sen2Cor](http://step.esa.int/main/third-party-plugins-2/sen2cor/).
 #' @param sen2cor_dir Path where sen2cor will be installed or searched
 #'  (by default it is a subdirectory `"sen2cor"` of the default sen2r directory).
 #' @param version (optional) Character: Sen2Cor version (one among
@@ -220,7 +220,7 @@ install_sen2cor <- function(
 #' @name link_sen2cor
 #' @rdname install_sen2cor
 #' @description `link_sen2cor()` links an existing standalone version of
-#'  [Sen2Cor](http://step.esa.int/main/third-party-plugins-2/sen2cor) to sen2r.
+#'  [Sen2Cor](http://step.esa.int/main/third-party-plugins-2/sen2cor/) to sen2r.
 #' @export
 link_sen2cor <- function(sen2cor_dir) {
 

diff --git a/R/list_indices.R b/R/list_indices.R
@@ -13,7 +13,7 @@
 #'  - `a`, `b`, `x`: parameter values (NA for non required parameters).
 #' @param pattern A regular expression on index names.
 #' @param all Logical: if TRUE, all the indices retrieved from 
-#'  [IDB](http://www.indexdatabase.de/db/is.php?sensor_id=96) are returned;
+#'  [IDB](https://www.indexdatabase.de/db/is.php?sensor_id=96) are returned;
 #'  if FALSE (default), only indices checked by the authors are returned.
 #' @return A data.frame with the required information. The table contains
 #'  also the following attributes:

diff --git a/R/s2_calcindices.R b/R/s2_calcindices.R
@@ -10,7 +10,7 @@
 #' @param indices Character vector with the names of the required
 #'  indices. Values should be included in names corresponding to the
 #'  Abbreviations of the following indices:
-#'  [IDB](http://www.indexdatabase.de/db/is.php?sensor_id=96).
+#'  [IDB](https://www.indexdatabase.de/db/is.php?sensor_id=96).
 #' @param outdir (optional) Full name of the output directory where
 #'  the files should be created (default: current directory).
 #'  `outdir` can bot be an existing or non-existing directory (in the

diff --git a/R/s2_download.R b/R/s2_download.R
@@ -9,9 +9,13 @@
 #' @param downloader Executable to use to download products
 #'  (default: "builtin"). Alternatives are "builtin" or "aria2"
 #'  (this requires aria2c to be installed).
-#' @param apihub Path of the "apihub.txt" file containing credentials
+#' @param apihub Path of the `apihub.txt` file containing credentials
 #'  of SciHub account.
 #'  If NA (default), the default location inside the package will be used.
+#' @param service Character: it can be `"dhus"` or `"apihub"`, in which cases
+#'  the required service is forced instead that the one present in the URLs
+#'  passed through argument `s2_prodlist`.
+#'  If NA (default), the service present in the URLs is maintained.
 #' @param tile Deprecated argument
 #' @param outdir (optional) Full name of the existing output directory
 #'  where the files should be created (default: current directory).
@@ -62,6 +66,7 @@ s2_download <- function(
   s2_prodlist = NULL,
   downloader = "builtin",
   apihub = NA,
+  service = NA,
   tile = NULL,
   outdir = ".",
   order_lta = TRUE,
@@ -77,6 +82,7 @@ s2_download <- function(
     s2_prodlist = s2_prodlist,
     downloader = downloader,
     apihub = apihub,
+    service = service,
     outdir = outdir,
     order_lta = order_lta,
     overwrite = overwrite,
@@ -91,15 +97,13 @@ s2_download <- function(
   s2_prodlist = NULL,
   downloader = "builtin",
   apihub = NA,
+  service = NA,
   outdir = ".",
   order_lta = TRUE,
   overwrite = FALSE,
   .s2_availability = NULL
 ) {
 
-  # to avoid NOTE on check
-  i <- mission <- level <- sensing_datetime <- id_orbit <- id_tile <- NULL
-
   # convert input NA arguments in NULL
   for (a in c("s2_prodlist", "apihub")) {
     if (suppressWarnings(all(is.na(get(a))))) {
@@ -117,8 +121,16 @@ s2_download <- function(
   # TODO add input checks
   s2_meta <- safe_getMetadata(s2_prodlist, info = "nameinfo")
 
-  # read credentials
-  creds <- read_scihub_login(apihub)
+  # check input server
+  s2_server <- ifelse(
+    grepl("^http.+Products\\(.+\\)/\\$value$", s2_prodlist),
+    "scihub",
+    ifelse(
+      grepl("^gs://gcp-public-data-sentinel-2", s2_prodlist),
+      "gcloud",
+      "unrecognised"
+    )
+  )
 
   # check downloader
   if (!downloader %in% c("builtin", "aria2", "aria2c")) {
@@ -142,6 +154,14 @@ s2_download <- function(
     downloader <- "builtin"
   }
 
+  # check outdir
+  if (!dir.exists(outdir)) {
+    print_message(
+      type = "error",
+      "Output directory does not exist."
+    )
+  }
+
   # Split products to be downloaded from products to be ordered
   s2_availability <- if (is.null(.s2_availability)) {
     print_message(
@@ -154,20 +174,88 @@ s2_download <- function(
     .s2_availability
   }
 
+  # Split products basing on download method
+  s2_toorder <- which(!s2_availability)
+  s2_todownload_scihub <- which(s2_availability & s2_server == "scihub")
+  s2_todownload_gcloud <- which(s2_server == "gcloud")
 
   # Order products stored from the Long Term Archive
   if (order_lta == TRUE) {
     ordered_products <- .s2_order(
-      s2_prodlist, 
-      .s2_availability = s2_availability, 
+      s2_prodlist[s2_toorder], 
+      .s2_availability = s2_availability[s2_toorder], 
       apihub = apihub
     )
   }
 
+  ## Download products available for download on SciHub
+  safe_prodlist <- .s2_download_scihub(
+    s2_prodlist = s2_prodlist[s2_todownload_scihub], 
+    s2_meta = s2_meta[s2_todownload_scihub,],
+    outdir = outdir, 
+    apihub = apihub, 
+    service = service,
+    downloader = downloader, 
+    overwrite = overwrite
+  )
 
-  ## Download products available for download
+  ## Download products available for download on GCloud
+  if (length(s2_todownload_gcloud) > 0) {
+    if (eval(parse(text = 'requireNamespace("sen2r.extras", quietly = TRUE)'))) {
+      safe_prodlist_gcloud <- eval(parse(text = paste0(
+        "sen2r.extras::.s2_download_gcloud(",
+        "  s2_prodlist = s2_prodlist[s2_todownload_gcloud],",
+        "  s2_meta = s2_meta[s2_todownload_gcloud,],",
+        "  outdir = outdir,",
+        "  overwrite = overwrite",
+        ")"
+      )))
+      safe_prodlist <- as(
+        c(safe_prodlist, safe_prodlist_gcloud)[
+          order(c(s2_todownload_scihub, s2_todownload_gcloud))
+          ],
+        "safelist"
+      )
+    }
+  }
 
-  safe_prodlist <- foreach(i = which(s2_availability), .combine = c) %do% {
+
+  return(safe_prodlist)
+
+}
+
+# internal function with the "core" download method from SciHub
+.s2_download_scihub <- function(
+  s2_prodlist, s2_meta, outdir, apihub, service, downloader, overwrite
+) {
+
+  # to avoid NOTE on check
+  i <- mission <- level <- sensing_datetime <- id_orbit <- id_tile <- NULL
+
+  # read credentials
+  if (length(s2_prodlist) > 0) {
+    creds <- read_scihub_login(apihub)
+  }
+
+  # check the used service
+  if (!service %in% c("apihub", "dhus", NA)) {
+    print_message(
+      type = "error",
+      "Argument 'service' can be only \"apihub\" or \"dhus\"; ",
+      "leaving the input URLs as are."
+    )
+  } else if (!is.na(service)) {
+    s2_prodlist <- gsub(
+      "^https://scihub.copernicus.eu/((apihub)|(dhus))/odata",
+      paste0("https://scihub.copernicus.eu/",service,"/odata"),
+      s2_prodlist
+    )
+  }
+
+  foreach(
+    i = seq_along(s2_prodlist), 
+    .combine = c
+  ) %do% {
 
     link <- s2_prodlist[i]
     zip_path <- file.path(outdir, paste0(names(s2_prodlist[i]),".zip"))
@@ -185,8 +273,8 @@ s2_download <- function(
       print_message(
         type = "message",
         date = TRUE,
-        "Downloading Sentinel-2 image ", which(i == which(s2_availability)),
-        " of ",sum(s2_availability)," (",basename(safe_path),")..."
+        "Downloading Sentinel-2 image ", i,
+        " of ",length(s2_prodlist)," (",basename(safe_path),")..."
       )
 
       if (downloader %in% c("builtin", "wget")) { # wget left for compatibility
@@ -299,8 +387,8 @@ s2_download <- function(
       print_message(
         type = "message",
         date = TRUE,
-        "Skipping Sentinel-2 image ", i," of ",which(i == which(s2_availability)),
-        " of ",sum(s2_availability),") ",
+        "Skipping Sentinel-2 image ", i,
+        " of ",length(s2_prodlist)," ",
         "since the corresponding folder already exists."
       )
 
@@ -315,7 +403,4 @@ s2_download <- function(
     as(setNames(link, safe_newname), "safelist")
 
   }
-
-  return(safe_prodlist)
-
 }
diff --git a/R/s2_gui.R b/R/s2_gui.R
@@ -3702,7 +3702,7 @@ s2_gui <- function(param_list = NULL,
           "<tr>",
           "<td style=\"padding-right: 10px;\">",
           as.character(
-            a(href="http://www.indexdatabase.de/db/is.php?sensor_id=96",
+            a(href="https://www.indexdatabase.de/db/is.php?sensor_id=96",
               target="_blank",
               img(
                 src="http://www.indexdatabase.de/daten/grafik/logo.png",