RC for v. 1.2.0 (#224)

Starting from this version sen2r supports ordering products from Long Term Archive (LTA) (see news at https://inthub.copernicus.eu/userguide/LongTermArchive). Now the user can automatically order SAFE products which are not available for direct download, and use them when made available. Some internal functions can be exploited to manually manage orders. Here above the related changes: ## New functions * `safe_is_valid()` to check if an order was processed; * `s2_order()` to order products from LTA. ## New arguments * `sen2r()` and `s2_download()` have a new argument `order_lta` (default: TRUE) to order SAFE archives not available for direct fownload; * `s2_gui()` has a new checkbox to set the previous argument. ## Other changes (not related with LTA) * Function `build_example_param_file()` does no more compute TOA and RGB432T (this to avoid downloading 2 SAFE archives). * Code coverage was expanded.
ranghetti · Oct 11, 2019 · be7b212 · be7b212
1 parent 9bb8e0a
commit be7b212
Show file tree

Hide file tree

Showing 108 changed files with 2,070 additions and 396 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -36,9 +36,6 @@ script:
   - R CMD build .
   - travis_wait 120 R CMD check *tar.gz
 
-after_success:
-    - Rscript -e 'covr::codecov(line_exclusions = c("R/s2_gui.R", "R/add_rgb_image.R", "R/geograbber_process.R", "R/editModPoly.R", "R/install_aria2.R", "R/ask_permission.R", "R/check_sen2r_deps.R", "R/create_indices_db.R", "R/helpers_extent.R", "R/create_indices_db.R", "R/gdal_formats_db.R", "R/path_check.R", "R/list_sen2r_paths.R", "R/check_param_list.R", "R/convert_datatype.R"))'
-
 env:
   global:
     - R_LIBS="http://cran.rstudio.com"

diff --git a/CRAN-RELEASE b/CRAN-RELEASE
@@ -1,2 +1,2 @@
-This package was submitted to CRAN on 2019-10-03.
-Once it is accepted, delete this file and tag the release (commit 62a869015a).
+This package was submitted to CRAN on 2019-10-07.
+Once it is accepted, delete this file and tag the release (commit 9bb8e0ae30).
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,7 +1,7 @@
 Package: sen2r
 Type: Package
 Title: Find, Download and Process Sentinel-2 Data
-Version: 1.1.0
+Version: 1.2.0
 Authors@R: c(person("Luigi", "Ranghetti",
                     email = "[email protected]",
                     role = c("aut", "cre"),

diff --git a/NAMESPACE b/NAMESPACE
@@ -30,12 +30,14 @@ export(s2_gui)
 export(s2_list)
 export(s2_mask)
 export(s2_merge)
+export(s2_order)
 export(s2_perc_masked)
 export(s2_rgb)
 export(s2_thumbnails)
 export(s2_tiles)
 export(s2_translate)
 export(safe_getMetadata)
+export(safe_is_online)
 export(safe_isvalid)
 export(safe_shortname)
 export(sen2cor)

diff --git a/NEWS.md b/NEWS.md
@@ -1,3 +1,25 @@
+# Version 1.2.0
+
+Starting from this version sen2r supports ordering products from Long Term Archive (LTA)
+(see news at https://inthub.copernicus.eu/userguide/LongTermArchive).
+Now the user can automatically order SAFE products which are not available for direct download, and use them when made available.
+Some internal functions can be exploited to manually manage orders.
+
+Here above the related changes:
+
+## New functions
+* `safe_is_valid()` to check if an order was processed;
+* `s2_order()` to order products from LTA.
+
+## New arguments
+* `sen2r()` and `s2_download()` have a new argument `order_lta` (default: TRUE) to order SAFE archives not available for direct fownload;
+* `s2_gui()` has a new checkbox to set the previous argument.
+
+## Other changes (not related with LTA)
+* Function `build_example_param_file()` does no more compute TOA and RGB432T (this to avoid downloading 2 SAFE archives).
+* Code coverage was expanded.
+
+
 # Version 1.1.0
 
 ## Changes in default values 

diff --git a/R/build_example_param_file.R b/R/build_example_param_file.R
@@ -48,6 +48,7 @@ build_example_param_file <- function(
         "s2_levels" = c("l1c", "l2a"),
         "sel_sensor" = c("s2a", "s2b"),
         "online" = TRUE,
+        "order_lta" = TRUE,
         "downloader" = "builtin",
         "overwrite_safe" = FALSE,
         "rm_safe" = "no",
@@ -58,9 +59,9 @@ build_example_param_file <- function(
         "extent" = system.file("extdata/vector/barbellino.geojson", package = "sen2r"),
         "s2tiles_selected" = NA,
         "s2orbits_selected" = NA,
-        "list_prods" = c("TOA", "BOA", "SCL"),
+        "list_prods" = c("BOA", "SCL"),
         "list_indices" = c("MSAVI2", "NDVI"),
-        "list_rgb" = c("RGB432B", "RGB432T", "RGB843B"),
+        "list_rgb" = c("RGB432B", "RGB843B"),
         "rgb_ranges" = list(
           c(0, 2500),
           c(0, 2500),

diff --git a/R/check_param_list.R b/R/check_param_list.R
@@ -76,7 +76,7 @@ check_param_list <- function(pm, type = "string", check_paths = FALSE, correct =
 
   # -- Parameters of length 1: check length
   pm_length1 <- c(
-    "preprocess", "online", "downloader", "overwrite_safe", "rm_safe",
+    "preprocess", "online", "order_lta", "downloader", "overwrite_safe", "rm_safe",
     "step_atmcorr", "max_cloud_safe", "timeperiod", "extent_name", "index_source",
     "mask_type", "max_mask", "mask_smooth", "mask_buffer", "clip_on_extent",
     "extent_as_mask", "reference_path", "res_s2", "unit", "proj", "resampling",
@@ -132,6 +132,9 @@ check_param_list <- function(pm, type = "string", check_paths = FALSE, correct =
   # -- online --
 
 
+  # -- order_lta --
+
+
   # -- downloader --
   if (!pm$downloader %in% c("builtin", "aria2")) {
     print_message(

diff --git a/R/s2_download.R b/R/s2_download.R
@@ -1,19 +1,23 @@
 #' @title Download S2 products.
-#' @description The function downloads a single S2 product.
-#'  Input filename must be an element obtained with
+#' @description The function downloads S2 products.
+#'  Input filenames must be elements obtained with
 #'  [s2_list] function
-#'  (the content must be a URL, and the name the product name).
-#' @param s2_prodlist List of the products to be downloaded
+#'  (each element must be a URL, and the name the product name).
+#' @param s2_prodlist Named character: list of the products to be downloaded
 #'  (this must be the output of [s2_list] function).
+#'  Alternatively, it can be the path of a JSON file exported by [s2_order].
 #' @param downloader Executable to use to download products
 #'  (default: "builtin"). Alternatives are "builtin" or "aria2"
 #'  (this requires aria2c to be installed).
 #' @param apihub Path of the "apihub.txt" file containing credentials
 #'  of SciHub account.
 #'  If NA (default), the default location inside the package will be used.
-#' @param tile Single Sentinel-2 Tile string (5-length character)
+#' @param tile Deprecated argument
 #' @param outdir (optional) Full name of the existing output directory
 #'  where the files should be created (default: current directory).
+#' @param order_lta Logical: if TRUE (default), products which are not available
+#'  for direct download are ordered from the Long Term Archive;
+#'  if FALSE, they are simply skipped.
 #' @param overwrite Logical value: should existing output archives be
 #'  overwritten? (default: FALSE)
 #' @return NULL (the function is called for its side effects)
@@ -22,6 +26,7 @@
 #' @author Lorenzo Busetto, phD (2019) \email{lbusett@@gmail.com}
 #' @note License: GPL 3.0
 #' @importFrom httr GET RETRY authenticate progress write_disk
+#' @importFrom foreach foreach "%do%"
 #' @export
 #'
 #' @examples
@@ -40,33 +45,69 @@
 #' #' # Download the whole product - using aria2
 #' s2_download(single_s2, outdir=tempdir(), downloader = "aria2")
 #'
-#' # Download a specific tile
-#' s2_download(single_s2, tile="32TQQ", outdir=tempdir())
-#' # (for products with compact names, the two above commands produce equivalent
-#' # results: the first one downloads a SAFE archive, while the second one
-#' # downloads single product files)
-#'
-#' # Download a serie of products
-#' pos <- st_sfc(st_point(c(12.0, 44.8)), crs=st_crs(4326))
-#' time_window <- as.Date(c("2017-05-01","2017-07-30"))
-#' example_s2_list <- s2_list(spatial_extent=pos, tile="32TQQ", time_interval=time_window)
-#' s2_download(example_s2_list, outdir=tempdir())
+#' # Download more products, ordering the ones stored in the Long Term Archive
+#' pos <- sf::st_sfc(sf::st_point(c(-57.8815,-51.6954)), crs = 4326)
+#' time_window <- as.Date(c("2018-02-21", "2018-03-20"))
+#' list_safe <- s2_list(spatial_extent = pos, time_interval = time_window)
+#' s2_download(list_safe, outdir=tempdir())
 #' }
 
-s2_download <- function(s2_prodlist = NULL,
-                        downloader = "builtin",
-                        apihub = NA,
-                        tile = NULL,
-                        outdir = ".",
-                        overwrite = FALSE) {
+s2_download <- function(
+  s2_prodlist = NULL,
+  downloader = "builtin",
+  apihub = NA,
+  tile = NULL,
+  outdir = ".",
+  order_lta = TRUE,
+  overwrite = FALSE
+) {
+
+  # warn for deprecated arguments
+  if (!missing("tile")) {
+    warning("argument 'tile' is deprecated and will not be used")
+  }
+
+  .s2_download(
+    s2_prodlist = s2_prodlist,
+    downloader = downloader,
+    apihub = apihub,
+    outdir = outdir,
+    order_lta = order_lta,
+    overwrite = overwrite,
+    .s2_availability = NULL
+  )
+
+}
+
+# internal function, used internally in order not to repeat the check
+# for online availability
+.s2_download <- function(
+  s2_prodlist = NULL,
+  downloader = "builtin",
+  apihub = NA,
+  outdir = ".",
+  order_lta = TRUE,
+  overwrite = FALSE,
+  .s2_availability = NULL
+) {
 
   # convert input NA arguments in NULL
-  for (a in c("s2_prodlist","tile","apihub")) {
+  for (a in c("s2_prodlist", "apihub")) {
     if (suppressWarnings(all(is.na(get(a))))) {
       assign(a,NULL)
     }
   }
 
+  # exit if empty
+  if (length(nn(s2_prodlist)) == 0) {
+    return(invisible(NULL))
+  }
+
+  # import s2_prodlist if it is a path
+  if (all(length(s2_prodlist) == 1, file.exists(s2_prodlist))) {
+    s2_prodlist <- unlist(fromJSON(s2_prodlist))
+  }
+
   # read credentials
   creds <- read_scihub_login(apihub)
 
@@ -80,7 +121,28 @@ s2_download <- function(s2_prodlist = NULL,
     downloader <- "builtin"
   }
 
-  for (i in seq_len(length(s2_prodlist))) {
+  # Split products to be downloaded from products to be ordered
+  s2_availability <- if (is.null(.s2_availability)) {
+    print_message(
+      type = "message",
+      date = TRUE,
+      "Check if products are available for download..."
+    )
+    safe_is_online(s2_prodlist)
+  } else {
+    .s2_availability
+  }
+
+
+  # Order products stored from the Long Term Archive
+  if (order_lta == TRUE) {
+    ordered_products <- .s2_order(s2_prodlist, .s2_availability = s2_availability)
+  }
+
+
+  ## Download products available for download
+
+  for (i in which(s2_availability)) {
 
     link <- s2_prodlist[i]
     zip_path <- file.path(outdir, paste0(names(s2_prodlist[i]),".zip"))
@@ -91,8 +153,8 @@ s2_download <- function(s2_prodlist = NULL,
       print_message(
         type = "message",
         date = TRUE,
-        "Downloading Sentinel-2 image ", i," of ",length(s2_prodlist),
-        " (",basename(safe_path),")..."
+        "Downloading Sentinel-2 image ", which(i == which(s2_availability)),
+        " of ",sum(s2_availability)," (",basename(safe_path),")..."
       )
 
       if (downloader %in% c("builtin", "wget")) { # wget left for compatibility
@@ -127,6 +189,16 @@ s2_download <- function(s2_prodlist = NULL,
         })
 
       }
+
+      # check if the user asked to download a LTA product
+      download_is_lta <- if (inherits(download, "response")) {
+        download$status_code == 202
+      } else if (inherits(download, "integer")) {
+        download == 22
+      } else FALSE
+      if (download_is_lta) {
+        # TODO
+      }
 
       if (inherits(download, "try-error")) {
         suppressWarnings(file.remove(zip_path))
@@ -142,8 +214,7 @@ s2_download <- function(s2_prodlist = NULL,
           sel_md5 <- httr::GET(
             url = gsub("\\$value$", "Checksum/Value/$value", as.character(link)),
             config = httr::authenticate(creds[1], creds[2]),
-            httr::write_disk(md5file <- tempfile(), overwrite = TRUE),
-            times = 10
+            httr::write_disk(md5file <- tempfile(), overwrite = TRUE)
           )
           md5 <- toupper(readLines(md5file, warn = FALSE)) == toupper(tools::md5sum(zip_path))
           file.remove(md5file)
@@ -177,8 +248,8 @@ s2_download <- function(s2_prodlist = NULL,
       print_message(
         type = "message",
         date = TRUE,
-        "Skipping Sentinel-2 image ", i," of ",length(s2_prodlist),
-        " (",basename(safe_path),") ",
+        "Skipping Sentinel-2 image ", i," of ",which(i == which(s2_availability)),
+        " of ",sum(s2_availability),") ",
         "since the corresponding folder already exists."
       )
 

diff --git a/R/s2_gui.R b/R/s2_gui.R
@@ -351,18 +351,26 @@ s2_gui <- function(param_list = NULL,
                   choiceNames = list("Online", "Offline"),
                   choiceValues = list(TRUE, FALSE),
                   selected = TRUE,
-                  inline = FALSE
+                  inline = TRUE
                 ),
 
                 # SciHub credentials
                 conditionalPanel(
                   condition = "input.online == 'TRUE'",
                   div(
                     style = "padding-bottom:10px;",
+                    checkboxInput(
+                      "make_lta_order",
+                      label = span(
+                        "Order from LTA\u2000",
+                        actionLink("help_lta_order", icon("question-circle"))
+                      ),
+                      value = TRUE
+                    ),
                     actionButton(
                       "scihub_md",
                       label = "\u2000Login to SciHub",
-                      icon=icon("user-circle")
+                      icon = icon("user-circle")
                     )
                   )
                 )
@@ -2633,6 +2641,37 @@ s2_gui <- function(param_list = NULL,
       ))
     })
 
+    observeEvent(input$help_lta_order, {
+      showModal(modalDialog(
+        title = "Order from LTA",
+        p(HTML(
+          "Starting from September 2019, SAFE archives older than 12 months",
+          "(Level-1C) or 18 months (Level-2A) are generally not available",
+          "for direct download, but must be ordered from the Long Term Archive",
+          "(see <a href='https://inthub.copernicus.eu/userguide/LongTermArchive'",
+          "target='_blank'>this page</a> for any details)."
+        )),
+        p(HTML(
+          "Checking this option, products which are not available for direct",
+          "download are ordered, so to be available at a later time.",
+          "There is no way to know when the will be made available; the user",
+          "can re-launch the same sen2r processing chain at a later time:",
+          "in this way, when missing SAFE archives will be made available",
+          "they will be downloaded and the output prodcut archive will be updated."
+        )),
+        p(HTML(
+          "Alternatively, specific non-interactive functions are available",
+          "to manage orders (see",
+          "<a href='https://sen2r.ranghetti.info/reference/safe_is_online.html'",
+          "target='_blank'><tt>safe_is_online()</tt></a> and",
+          "<a href='https://sen2r.ranghetti.info/reference/s2_order.html'",
+          "target='_blank'><tt>s2_order()</tt></a>)."
+        )),
+        easyClose = TRUE,
+        footer = NULL
+      ))
+    })
+
     observeEvent(input$help_downloader, {
       showModal(modalDialog(
         title = "Downloader",
@@ -3363,6 +3402,7 @@ s2_gui <- function(param_list = NULL,
       rl$s2_levels <- c(if(safe_req$l1c==TRUE){"l1c"}, if(safe_req$l2a==TRUE){"l2a"}) # required S2 levels ("l1c","l2a")
       rl$sel_sensor <- input$sel_sensor # sensors to use ("s2a", "s2b")
       rl$online <- as.logical(input$online) # TRUE if online mode, FALSE if offline mode
+      rl$order_lta <- as.logical(input$make_lta_order) # TRUE to order from LTA, FALSE to skip
       rl$downloader <- input$downloader # downloader ("builtin" or "aria2")
       rl$overwrite_safe <- as.logical(input$overwrite_safe) # TRUE to overwrite existing SAFE, FALSE not to
       rl$rm_safe <- input$rm_safe # "yes" to delete all SAFE, "l1c" to delete only l1c, "no" not to remove
@@ -3519,6 +3559,7 @@ s2_gui <- function(param_list = NULL,
         updateCheckboxGroupInput(session, "list_levels", selected = pl$s2_levels)
         updateCheckboxGroupInput(session, "sel_sensor", selected = pl$sel_sensor)
         updateRadioButtons(session, "online", selected = pl$online)
+        updateRadioButtons(session, "make_lta_order", selected = pl$order_lta)
         updateRadioButtons(session, "downloader", selected = pl$downloader)
         updateRadioButtons(session, "overwrite_safe", selected = pl$overwrite_safe)
         updateRadioButtons(session, "rm_safe", selected = pl$rm_safe)
@@ -3692,7 +3733,7 @@ s2_gui <- function(param_list = NULL,
         sendSweetAlert(
           session, NULL,
           paste0(
-            "Please select at least one product, spectral index or RGB image",
+            "Please select at least one product, spectral index or RGB image ",
             "before continuing."
           ),
           type = "error"