Merge branch 'grass8' into renovate/super-linter-super-linter-7.x

.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
                 .*\.svg$
           )
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.41.0
+    rev: v0.42.0
     hooks:
       - id: markdownlint-fix
   # Using this mirror lets us use mypyc-compiled black, which is about 2x faster

src/raster/r.buildvrt.gdal/Makefile

@@ -0,0 +1,7 @@
+MODULE_TOPDIR = ../..
+
+PGM=r.buildvrt.gdal
+
+include $(MODULE_TOPDIR)/include/Make/Script.make
+
+default: script

src/raster/r.buildvrt.gdal/r.buildvrt.gdal.html

@@ -0,0 +1,67 @@
+<h2>DESCRIPTION</h2>
+
+<em>r.buildvrt.gdal</em> builds GDAL virtual rasters over GRASS GIS raster
+maps and links them to the mapset with <em>r.external</em>. The module is
+written as a workaround for a limitation in GRASS GIS Virtual Rasters (VRT)
+format with GDAL-linked raster maps (through <em>r.external</em> /
+<em>r.external.out</em>. In that case GRASS GIS Virtual Rasters currently
+show performance issues. See:
+<a href="https://github.com/OSGeo/grass/issues/4345">#4345</a>
+
+<p>
+For the resulting maps GDAL VRT text files are created either in a
+directory named "gdal" in the current mapset or in a user-defined <b>
+vrt_directory</b>. Those files are not removed when the raster map is
+removed and the user is responsible for removing them when needed.
+
+<h2>REQUIREMENTS</h2>
+<em>r.buildvrt.gdal</em> uses the Python bindings for
+<a href="https://pypi.org/project/GDAL">GDAL</a> and requires the
+GDAL-GRASS driver to include raster maps in native GRASS format in
+GDAL VRTs.
+
+<h2>EXAMPLES</h2>
+<div class="code"><pre>
+# Create external example data
+regs='s,0,1000
+n,500,1500'
+
+eval `g.gisenv`
+external_path="${GISDBASE}/${LOCATION}/${MAPSET}/.tmp/vrt"
+mkdir -p "$external_path"
+for reg in $regs
+do
+  r=$(echo $reg | cut -f1 -d",")
+  s=$(echo $reg | cut -f2 -d",")
+  n=$(echo $reg | cut -f3 -d",")
+
+  g.region -g n=$n s=$s w=0 e=1000 res=1
+  r.external.out format=GTiff options="compress=LZW,PREDICTOR=3" \
+    directory="$external_path"
+  r.mapcalc --o --v expression="${r}_${s}_gtiff_ntfs=float(x()*y())"
+done
+
+# Run performance tests
+g.region -g n=1500 s=0 w=0 e=1000 res=1
+format_type=gtiff_ntfs
+rmaps=$(g.list type=raster pattern="*_*_${format_type}", sep=",")
+
+# Using GRASS GIS VRT
+r.buildvrt --o --v input="$rmaps" output=vrt_${format_type}
+time r.univar map=vrt_${format_type}
+
+# Using GDAL VRT
+r.buildvrt.gdal --o --v input="$rmaps" output=vrt_${format_type}_gdal
+time r.univar map=vrt_${format_type}_gdal
+</pre></div>
+
+<h2>SEE ALSO</h2>
+<em>
+<a href="https://grass.osgeo.org/grass-stable/manuals/r.buildvrt.html">r.buildvrt</a>,
+<a href="https://grass.osgeo.org/grass-stable/manuals/r.patch.html">r.patch</a>,
+<a href="https://grass.osgeo.org/grass-stable/manuals/r.external.html">r.external</a>,
+<a href="https://grass.osgeo.org/grass-stable/manuals/r.external.out.html">r.external.out</a>
+</em>
+
+<h2>AUTHORS</h2>
+Stefan Blumentrath

src/raster/r.buildvrt.gdal/r.buildvrt.gdal.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+
+"""
+ MODULE:       r.buildvrt.gdal
+ AUTHOR(S):    Stefan Blumentrath
+ PURPOSE:      Build GDAL Virtual Rasters (VRT) over GRASS GIS raster maps
+ COPYRIGHT:    (C) 2024 by stefan.blumentrath, and the GRASS Development Team
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+"""
+
+# %module
+# % description: Build GDAL Virtual Rasters (VRT) over GRASS GIS raster maps
+# % keyword: raster
+# % keyword: virtual
+# % keyword: gdal
+# % keyword: patch
+# %end
+
+# %option G_OPT_R_INPUTS
+# % key: input
+# % type: string
+# % required: no
+# % multiple: yes
+# %end
+
+# %option G_OPT_F_INPUT
+# % key: file
+# % required: no
+# %end
+
+# %option G_OPT_M_DIR
+# % key: vrt_directory
+# % description: Directory to store GDAL VRT files in. Default is: $GISDBASE/$PROJECT/$MAPSET/gdal
+# % required: no
+# %end
+
+# %option G_OPT_R_OUTPUT
+# %end
+
+# %flag
+# % key: m
+# % label: Read data range from metadata
+# % description: WARNING: metadata are sometimes approximations with wrong data range
+# %end
+
+# %flag
+# % key: r
+# % label: Create fast link without data range
+# % description: WARNING: some modules do not work correctly without known data range
+# %end
+
+# %rules
+# % required: input,file
+# % exclusive: input,file
+# % exclusive: -m,-r
+# %end
+
+
+import json
+import sys
+
+from pathlib import Path
+
+import grass.script as gs
+
+
+def get_raster_gdalpath(
+    map_name, check_linked=True, has_grasadriver=False, gis_env=None
+):
+    """Get the GDAL-readable path to a GRASS GIS raster map
+
+    Returns either the link stored in the GDAL-link file in the cell_misc
+    directory for raster maps linked with r.external or r.external.out
+    - if requested - or the path to the header of the GRASS GIS raster
+    map"""
+    if check_linked:
+        # Check GDAL link header
+        map_info = gs.find_file(map_name)
+        header_path = (
+            Path(gis_env["GISDBASE"])
+            / gis_env["LOCATION_NAME"]
+            / map_info["mapset"]
+            / "cell_misc"
+            / map_info["name"]
+            / "gdal"
+        )
+        if header_path.is_file():
+            gdal_path = Path(
+                gs.parse_key_val(header_path.read_text(), sep=": ")["file"]
+            )
+            if gdal_path.exists():
+                return str(gdal_path)
+
+    # Get native GRASS GIS format header
+    if not has_grasadriver:
+        gs.fatal(
+            _(
+                "The GDAL-GRASS GIS driver is unavailable. "
+                "Cannot create GDAL VRTs for map <{}>. "
+                "Please install the GDAL-GRASS plugin."
+            ).format(map_name)
+        )
+
+    gdal_path = Path(gs.find_file(map_name)["file"].replace("/cell/", "/cellhd/"))
+    if gdal_path.is_file():
+        return gdal_path
+
+    # Fail if file path cannot be determined
+    gs.fatal(_("Cannot determine GDAL readable path to raster map {}").format(map_name))
+
+
+def main():
+    """run the main workflow"""
+    options, flags = gs.parser()
+
+    # lazy imports
+    global gdal
+    try:
+        from osgeo import gdal
+    except ImportError:
+        gs.fatal(
+            _(
+                "Unable to load GDAL Python bindings (requires "
+                "package 'python-gdal' or Python library GDAL "
+                "to be installed)."
+            )
+        )
+
+    # Check if GRASS GIS driver is available
+    has_grassdriver = True
+    if not gdal.GetDriverByName("GRASS"):
+        has_grassdriver = False
+
+    # Get GRASS GIS environment info
+    gisenv = gs.gisenv()
+
+    # Get inputs
+    if options["input"]:
+        inputs = options["input"].split(",")
+    else:
+        inputs = Path(options["file"]).read_text(encoding="UTF8").strip().split("\n")
+
+    if len(inputs) < 1:
+        gs.fatal(_("At least one input map is required".format(inputs[0])))
+
+    inputs = [get_raster_gdalpath(raster_map, gis_env=gisenv) for raster_map in inputs]
+
+    # Get output
+    output = options["output"]
+
+    # Create a directory to place GDAL VRTs in
+    if options["vrt_directory"]:
+        vrt_dir = Path(options["vrt_directory"])
+    else:
+        vrt_dir = Path(gisenv["GISDBASE"]).joinpath(
+            gisenv["LOCATION_NAME"], gisenv["MAPSET"], "gdal"
+        )
+    vrt_dir.mkdir(exist_ok=True, parents=True)
+
+    # Create GDAL VRT
+    vrt_path = str(vrt_dir / f"{output}.vrt")
+    gs.verbose(_("Creating GDAL VRT '{}'.").format(vrt_path))
+    gdal.BuildVRT(vrt_path, inputs)
+
+    # Import (link) GDAL VRT
+    gs.run_command(
+        "r.external",
+        quiet=True,
+        flags=f"oa{''.join([key for key, val in flags.items() if val])}",
+        input=str(vrt_path),
+        output=output,
+    )
+    gs.raster_history(output, overwrite=True)
+
+
+if __name__ == "__main__":
+
+    sys.exit(main())