merge

Merge branch 'main' into reproj_xy_list

# Conflicts:
#	.github/workflows/check-standard.yaml
#	man/vapour_srs_wkt.Rd
#	man/vapour_vrt.Rd
#	man/vapour_warp_raster.Rd

.Rbuildignore

@@ -31,3 +31,4 @@
 ^CRAN-SUBMISSION$
 ^Dockerfile$
 ^inst/ci$
+

.github/workflows/R-CMD-check.yaml → .github/workflows/check-standard.yaml

@@ -2,8 +2,8 @@
 # Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
 on:
   push:
-    branches: [main, reproj_xy_list]
-  pull_request:
+   branches: [main, dev-main]
+ pull_request:
     branches: [main]
 
 name: R-CMD-check
@@ -18,9 +18,10 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - {os: macOS-latest,   r: 'release'}
-          - {os: windows-latest, r: 'release', binary: 'rtools'}
-          - {os: windows-latest, r: 'release', binary: 'rwinlib'}
+          - {os: macos-latest,   r: 'release'}
+          - {os: windows-latest, r: '4.2.3', rtools-version: '42', windows-path-include-rtools: 'true', update-rtools: 'true', binary: 'rwinlib'}
+          - {os: windows-devel, r: '4.3.0', rtools-version: '43', windows-path-include-rtools: 'true', update-rtools: 'true'}
+
           - {os: ubuntu-latest,   r: 'devel', http-user-agent: 'release'}
           - {os: ubuntu-latest,   r: 'release'}
           - {os: ubuntu-latest,   r: 'oldrel-1'}
@@ -38,7 +39,8 @@ jobs:
            brew install geos
            brew install gdal
     
-      - uses: actions/checkout@v2
+
+      - uses: actions/checkout@v3
 
       - uses: r-lib/actions/setup-pandoc@v2
 
@@ -47,18 +49,14 @@ jobs:
           r-version: ${{ matrix.config.r }}
           http-user-agent: ${{ matrix.config.http-user-agent }}
           use-public-rspm: true
-          rtools-version: '42'
-
+          rtools-version: ${{ matrix.config.rtools.version }}
+          windows-path-include-rtools: ${{ matrix.config.windows-path-include-rtools }}
+
       - uses: r-lib/actions/setup-r-dependencies@v2
         with:
           extra-packages: any::rcmdcheck
           needs: check
 
-      - name: ZapRtools
-        if: runner.os == 'windows' && matrix.config.binary == 'rwinlib'
-        run: rm src/Makevars.ucrt
-
-
       - uses: r-lib/actions/check-r-package@v2
         with:
           upload-snapshots: true

.gitignore

@@ -17,3 +17,4 @@ proj_conf_test.c
 proj_conf_test.cpp
 configure*
 CRAN-RELEASE
+*.gpkg

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: vapour
 Title: Access to the 'Geospatial Data Abstraction Library' ('GDAL')
-Version: 0.9.2
+Version: 0.9.5.9003
 Authors@R: c(person("Michael", "Sumner", email = "[email protected]", role = c("aut", "cre"), comment = c(ORCID = "0000-0002-2471-7511")), 
              person("Simon", "Wotherspoon", role = "ctb", comment = "RasterIO configuration for resampling options"), 
              person("Mark", "Padgham", role = "ctb", comment = "helped get started :)"), 
@@ -12,7 +12,8 @@ Authors@R: c(person("Michael", "Sumner", email = "[email protected]", role = c(
              person("Dale", "Maschette", role = c("ctb"), comment = "created the hex logo"), 
              person("Joseph", "Stachelek", role = "ctb"), 
              person("Even", "Rouault", role = "ctb", comment = "primary author of the COG format and its use of the GDALwarp app-library, example code used by the warper function here"), 
-             person("Robert", "Hijmans", role = "ctb", comment = "code in terra package used as example/inspiration"))
+             person("Robert", "Hijmans", role = "ctb", comment = "code in terra package used as example/inspiration"), 
+             person("Dewey", "Dunnington", role = "ctb", comment = "wrote the columnar-access mode streaming Arrow support"))
 Description: Provides low-level access to 'GDAL' functionality.  
   'GDAL' is the 'Geospatial Data Abstraction Library' a translator for raster and vector geospatial data formats 
   that presents a single raster abstract data model and single vector abstract data model to the calling application 
@@ -25,6 +26,7 @@ LazyData: true
 LinkingTo: Rcpp
 Imports:
     jsonlite,
+    nanoarrow,
     Rcpp,
     utils
 RoxygenNote: 7.2.3
@@ -35,7 +37,7 @@ Suggests:
     markdown,
     rmarkdown,
     spelling
-SystemRequirements: libgdal-dev, GDAL (>= 2.2.3), PROJ (>= 4.8.0), C++11
+SystemRequirements: libgdal-dev, GDAL (>= 2.2.3), PROJ (>= 4.8.0)
 VignetteBuilder: knitr
 URL: https://github.com/hypertidy/vapour
 BugReports: https://github.com/hypertidy/vapour/issues

NAMESPACE

@@ -1,7 +1,12 @@
 # Generated by roxygen2: do not edit by hand
 
+export(gdal_raster_data)
+export(gdal_raster_dsn)
+export(gdal_raster_image)
 export(vapour_all_drivers)
 export(vapour_create)
+export(vapour_create_options)
+export(vapour_crs_is_lonlat)
 export(vapour_driver)
 export(vapour_gdal_version)
 export(vapour_geom_name)

NEWS.md

@@ -1,3 +1,56 @@
+# vapour dev
+
+* `vapour_vrt()` called with geolocation arrays now scrubs the geotransform from the output (which ensures the
+ warper uses the arrays without setting '-geoloc'). 
+
+* `vapour_create()` gets creation options, data type options, and driver options. 
+
+* Fixed failure to close file created by `vapour_create()` fixes #202. 
+
+# vapour 0.9.5
+
+* `vapour_vrt()` gains an 'overview' argument. Wish of https://github.com/hypertidy/vapour/issues/186
+
+* `gdal_raster_data()`, `gdal_raster_dsn()`, and `gdal_raster_image()` now allow a source with geolocation arrays to be warped to a non-longlat projection. 
+
+* The extent reported by 'vapour_raster_info()' is now correct for the general case including non-zero skew geotransforms. 
+
+* 'vapour_raster_info()' gains a  'corners' element. 
+
+* All functions that read or query data source/s now check for tilde "~" at the beginning of the string/s, and normalize the path as needed. Reported by @Sibada in #193. 
+
+* vapour now imports nanoarrow and includes internal experimental support for GDAL (>= 3.6) stream reading (RFC 86). Nothing is
+exposed for general use yet.
+
+* New Makevars.ucrt with patch contributed by CRAN, Tomas Kalibera. 
+
+* raster read gains "unscale" argument, `TRUE` by default which means offset and scale values are applied and return value is of type Float64, set to `FALSE` to avoid applying scale/offset to band values (this was always implicitly false (no offset scale applied) if the output type wasn't suitable). . 
+
+* Fix inefficiency in gdal_raster_data() which was warping all bands in MEM lol. 
+
+* Fix for `vapour_vrt()` where 'geolocation' was not being included. Fixes #192. 
+
+* New capability to drop dataset and band metdata, used by functions `vapour_vrt()` and `vapour_warp_raster()` which gain a new argument 'nomd', which
+is FALSE by default. If TRUE, the dataset and band metadata are removed from an open dataset before it is converted to VRT. 
+
+This particularly makes VRT DSN strings a lot smaller, for use by whatarelief and raadtools for example.  There should be no change to current default uses, the argument 'nomd' must be specified for the change. 
+
+
+* New function `vapour_crs_is_lonlat()` to test crs string. 
+
+* Removed C++11 requirement. 
+
+* remove LDFLAGS from configure, as per sf #1369 and vapour #188, thanks to @gremms1 and @sgoslee for 
+ report on Fedora. 
+
+* Removed FromHandle (introduced GDAL 2.3) and undeclared use of down_cast, and now tested on GDAL 2.2.3. 
+
+* New Makevars.ucrt with patch contributed by CRAN, and removed outdated C++11 requirement.
+
+# vapour 0.9.3
+
+* Update to CRAN Makevars.ucrt. 
+
 # vapour 0.9.2
 
 * Resubmit after 0.9.1 was rejected, fixed configure.ac and checked on on M1MAC/macbuilder. 

R/00_read_block.R

@@ -1,12 +1,40 @@
+#' @name vapour_create
+#' @export
+vapour_create_options <- function(driver = "GTiff") {
+
+
+    if (driver[1] == "GTiff")  {
+      default_options = c(
+  "SPARSE_OK=YES", 
+  "TILED=YES", "BLOCKXSIZE=512", "BLOCKYSIZE=512", 
+  "COMPRESS=DEFLATE", "BIGTIFF=IF_SAFER") 
+    } else if (driver[1] == "COG") {
+      default_options <- c(
+        "SPARSE_OK=YES", 
+         "BLOCKSIZE=512", 
+        "COMPRESS=DEFLATE", "BIGTIFF=IF_SAFER") 
+    } else {
+      default_options <- character(0)
+    }
+  default_options
+}
+
+
 #' Create raster file
 #'
 #' This is in an incomplete interface to raster writing, for exploring. 
 #' 
 #' If GeoTIFF is used (`driver = "GTiff"`, recommended) then the output is tiled 512x512, and has DEFLATE compression, and
 #' is sparse when created (no values are initiated, so the file is tiny). 
 #' 
-#' Note that there is no restriction on where you can read or write from, the responsibility is yours. In future we will 
-#' allow control of output tiling and data type etc. 
+#' Note that there is no restriction on where you can read or write from, the responsibility is yours. There is no auto driver detection
+#' done for the file format, it's up to you to set the file extension _and_ the driver. 
+#' 
+#' File is created using CreateCopy from a VRT in memory. This is so that we can instantiate COG layer with 'driver = "COG"'. 
+#' Please note that performance is best for GTiff itself, with 'SPARSE_OK=YES'. We don't yet know how to instantiate a large
+#' COG with overviews. 
+#' 
+#' There are default creation options set for COG and GTiff drivers, see 'vapour_create_options(driver "GTiff")' for what those are. 
 #' 
 #' @param filename filename/path to create
 #' @param driver GDAL driver to use (GTiff is default, and recommended)
@@ -15,6 +43,8 @@
 #' @param projection projection of the output, best to use a full WKT but any string accepted
 #' @param n_bands number of bands in the output, default is 1
 #' @param overwrite not TRUE by default
+#' @param datatype the name of a GDAL datatype ('Float32', 'Int64', etc)
+#' @param options character vector of creation of options for the driver in use `c('COMPRESS=DEFLATE')` note how these are constructed (no '-co' element)
 #'
 #' @return the file path that was created
 #' @export
@@ -28,11 +58,30 @@
 #'  file.remove(tfile)
 #' }
 vapour_create <- function(filename, driver = "GTiff", extent = c(-180, 180, -90, 90), 
-                          dimension = c(2048, 1024), projection = "OGC:CRS84", n_bands = 1L, overwrite = FALSE) {
+                          dimension = c(2048, 1024), projection = "OGC:CRS84", n_bands = 1L, overwrite = FALSE, 
+                          datatype = "Float32",
+                          options = vapour_create_options(driver)) {
 
   if (!overwrite && file.exists(filename)) stop("'filename' exists")
 
-  vapour_create_cpp(filename, driver, extent, dimension, projection, n_bands)
+  driver <- driver[1L]
+  if (length(driver) < 1 || !nzchar(driver) || is.na(driver)) {
+    stop("driver name is not valid")
+  }
+  stopifnot(is.numeric(extent))
+  stopifnot(length(extent) == 4L)
+  df <- diff(extent)[c(1L, 3L)]
+  if(!df[1] > 0) stop("extent is not valid, must be c(xmin, xmax, ymin, ymax) :  xmax !> xmin")
+  if(!df[2] > 0) stop("extent is not valid, must be c(xmin, xmax, ymin, ymax) :  ymax !> ymin")
+
+  if (is.null(options)) {
+    options <- vapour_create_options(driver)
+  } 
+  if (!is.character(options)) options <- character()
+  if (length(options) < 1) options <- character()
+  if (!nzchar(options[1])) options <- character()
+  if (is.na(options[1])) options <- character()
+  vapour_create_cpp(filename, driver, extent, dimension, projection, n_bands, datatype, options)
 }
 
 
@@ -80,22 +129,24 @@ vapour_create_copy <- function(dsource, filename, overwrite = FALSE, driver = "G
 #' @param dimension window size to read from, or write to
 #' @param band_output_type numeric type of band to apply (else the native type if '') can be one of 'Byte', 'Int32', or 'Float64'
 #' @param band which band to read (1-based)
+#' @param unscale default is `TRUE` so native values will be converted by offset and scale to floating point
 #'
 #' @return a list with a vector of data from the band read
 #' @export
 #'
 #' @examples
 #' f <- system.file("extdata", "sst.tif", package = "vapour")
 #' v <- vapour_read_raster_block(f, c(0L, 0L), dimension = c(2L, 3L), band = 1L)
-vapour_read_raster_block <- function(dsource, offset, dimension, band = 1L, band_output_type = "") {
+vapour_read_raster_block <- function(dsource, offset, dimension, band = 1L, band_output_type = "", unscale = TRUE) {
   dsource <- .check_dsn_single(dsource)
   if (anyNA(band) || length(band) < 1L) stop("missing band value")
   if (file.exists(dsource)) {
     dsource <- normalizePath(dsource)
   }
   vapour_read_raster_block_cpp(dsource, as.integer(rep(offset, length.out = 2L)),
                                as.integer(rep(dimension, length.out = 2L)), band = as.integer(band[1L]),
-                               band_output_type = band_output_type)
+                               band_output_type = band_output_type, 
+                               unscale = unscale)
 }
 #' Write data to a block *in an existing file*.
 #'

R/00_utils.R

@@ -18,11 +18,14 @@ gdalinfo_internal <- function(x, json = TRUE,
   }
   if (length(sd) > 1) message("'sd' argument cannot be vectorized over 'dsn', ignoring all but first value")
 
+  version <- vapour_gdal_version()
+  v3 <- TRUE
+  if (grepl("GDAL 2", version )) v3 <- FALSE
   extra <- c(if(json) "-json",
              if (is.numeric(sd) && sd[1L] > 0) c("-sd", sd[1L]),
              if (stats) "-stats",
              if (checksum) "-checksum",
-             if (nchar(wkt_format[1]) > 0) c("-wkt_format", wkt_format[1L]),
+             if (nchar(wkt_format[1]) > 0 && v3) c("-wkt_format", wkt_format[1L]),
              if (length(oo) > 0 && any(nchar(oo) > 0) ) rep_zip("-oo", oo[nchar(oo) > 0]),
              if (length(initial_format) > 0 && any(nchar(initial_format) > 0)) rep_zip("-if", initial_format[nchar(initial_format) > 0]))