diff --git a/docs/src/examples/HLS_Example.ipynb b/docs/src/examples/HLS_Example.ipynb index d92e239..ff23ff8 100644 --- a/docs/src/examples/HLS_Example.ipynb +++ b/docs/src/examples/HLS_Example.ipynb @@ -6,9 +6,14 @@ "source": [ "# TiTiler-CMR: HLS Data Demo\n", "\n", + "The Harmonized Landsat Sentinel-2 dataset is available in two collections in CMR. This example will use data from the `HLSL30.002` (Landsat) dataset.\n", + "\n", "#### Requirements\n", - "- folium\n", - "- httpx\n", + "To run some of the chunks in this notebook you will need to install a few packages:\n", + "\n", + "- `earthaccess`\n", + "- `folium`\n", + "- `httpx`\n", "\n", "`!pip install folium httpx earthaccess`" ] @@ -16,14 +21,18 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ - "import json\n", - "import httpx\n", "import earthaccess\n", + "import geojson_pydantic\n", + "import httpx\n", + "import json\n", + "\n", "\n", - "from folium import Map, TileLayer" + "from folium import GeoJson, Map, TileLayer" ] }, { @@ -32,17 +41,15 @@ "metadata": {}, "outputs": [], "source": [ - "titiler_endpoint = \"http://localhost:8000\"" + "titiler_endpoint = \"http://localhost:8081\"" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "datasets = earthaccess.search_datasets(doi=\"10.5067/HLS/HLSL30.002\")\n", - "ds = datasets[0]" + "## Identify the dataset\n", + "You can find the `HLSL30.002` dataset using the earthaccess.search_datasets function." ] }, { @@ -51,8 +58,21 @@ "metadata": {}, "outputs": [], "source": [ + "datasets = earthaccess.search_datasets(doi=\"10.5067/HLS/HLSL30.002\")\n", + "ds = datasets[0]\n", + "\n", "concept_id = ds[\"meta\"][\"concept-id\"]\n", - "print(\"Concept-Id: \", concept_id)" + "print(\"Concept-Id: \", concept_id)\n", + "print(\"Abstract: \", ds[\"umm\"][\"Abstract\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Examine a granule\n", + "\n", + "Each granule contains the data for a single point in time for an MGRS tile. " ] }, { @@ -84,12 +104,12 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "print(results[0])" + "## Demonstrate `assets_for_tile` method\n", + "\n", + "While rendering `xyz` tile images, `titiler-cmr` searches for assets using the `assets_for_tile` method which converts the `xyz` tile extent into a bounding box." ] }, { @@ -115,20 +135,10 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "with CMRBackend() as backend:\n", - " assets = backend.assets_for_tile(\n", - " x=62,\n", - " y=44,\n", - " z=7,\n", - " bands_regex=\"B[0-9][0-9]\",\n", - " concept_id=\"C2021957657-LPCLOUD\",\n", - " temporal=(\"2024-02-11\", \"2024-02-13\")\n", - " )\n" + "## `titiler.cmr` API documentation" ] }, { @@ -141,6 +151,15 @@ "IFrame(f\"{titiler_endpoint}/api.html\", 900,500)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Display tiles in an interactive map\n", + "\n", + "The `/tilejson.json` endpoint will provide a parameterized `xyz` tile URL that can be added to an interactive map." + ] + }, { "cell_type": "code", "execution_count": null, @@ -166,7 +185,6 @@ " (\"bands\", \"B02\"),\n", " # The data is in type of Uint16 so we need to apply some\n", " # rescaling/color_formula in order to create PNGs\n", - " # (\"rescale\", \"0,1000\"),\n", " (\"color_formula\", \"Gamma RGB 3.5 Saturation 1.7 Sigmoidal RGB 15 0.35\"),\n", " # We need to set min/max zoom because we don't want to use lowerzoom level (e.g 0)\n", " # which will results in useless large scale query\n", @@ -187,7 +205,7 @@ "bounds = r[\"bounds\"]\n", "m = Map(\n", " location=(47.590266824611675, -91.03729840730689),\n", - " zoom_start=r[\"maxzoom\"] - 3\n", + " zoom_start=r[\"maxzoom\"] - 2\n", ")\n", "\n", "TileLayer(\n", @@ -198,6 +216,152 @@ "m" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Render NDVI using the `expression` parameter\n", + "The `expression` parameter can be used to render images from an expression of a combination of the individual `bands`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "r = httpx.get(\n", + " f\"{titiler_endpoint}/WebMercatorQuad/tilejson.json\",\n", + " params = (\n", + " (\"concept_id\", concept_id),\n", + " # Datetime in form of `start_date/end_date`\n", + " (\"datetime\", \"2024-06-20T00:00:00Z/2024-06-27T23:59:59Z\"),\n", + " # We know that the HLS collection dataset is stored as File per Band\n", + " # so we need to pass a `band_regex` option to assign `bands` to each URL\n", + " (\"bands_regex\", \"B[0-9][0-9]\"),\n", + " # titiler-cmr can work with both Zarr and COG dataset\n", + " # but we need to tell the endpoints in advance which backend\n", + " # to use\n", + " (\"backend\", \"rasterio\"),\n", + " # NDVI\n", + " (\"expression\", \"(B05-B04)/(B05+B04)\"),\n", + " # Need red (B04) and nir (B05) for NDVI\n", + " (\"bands\", \"B05\"),\n", + " (\"bands\", \"B04\"),\n", + " # The data is in type of Uint16 so we need to apply some\n", + " # rescaling/color_formula in order to create PNGs\n", + " (\"colormap_name\", \"viridis\"),\n", + " (\"rescale\", \"-1,1\"),\n", + " # We need to set min/max zoom because we don't want to use lowerzoom level (e.g 0)\n", + " # which will results in useless large scale query\n", + " (\"minzoom\", 8),\n", + " (\"maxzoom\", 13),\n", + " )\n", + ").json()\n", + "\n", + "m = Map(\n", + " location=(47.9221313337365, -91.65432884883238),\n", + " zoom_start=r[\"maxzoom\"] - 1\n", + ")\n", + "\n", + "\n", + "TileLayer(\n", + " tiles=r[\"tiles\"][0],\n", + " opacity=1,\n", + " attr=\"NASA\",\n", + ").add_to(m)\n", + "\n", + "GeoJson(geojson).add_to(m)\n", + "\n", + "m" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## GeoJSON Statistics\n", + "The `/statistics` endpoint can be used to get summary statistics for a geojson `Feature` or `FeatureCollection`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "geojson = {\n", + " \"type\": \"FeatureCollection\",\n", + " \"features\": [\n", + " {\n", + " \"type\": \"Feature\",\n", + " \"properties\": {},\n", + " \"geometry\": {\n", + " \"coordinates\": [\n", + " [\n", + " [\n", + " -91.65432884883238,\n", + " 47.9221313337365\n", + " ],\n", + " [\n", + " -91.65432884883238,\n", + " 47.86503396133904\n", + " ],\n", + " [\n", + " -91.53842043960762,\n", + " 47.86503396133904\n", + " ],\n", + " [\n", + " -91.53842043960762,\n", + " 47.9221313337365\n", + " ],\n", + " [\n", + " -91.65432884883238,\n", + " 47.9221313337365\n", + " ]\n", + " ]\n", + " ],\n", + " \"type\": \"Polygon\"\n", + " }\n", + " }\n", + " ]\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "r = httpx.post(\n", + " f\"{titiler_endpoint}/statistics\",\n", + " params=(\n", + " (\"concept_id\", concept_id),\n", + " # Datetime in form of `start_date/end_date`\n", + " (\"datetime\", \"2024-07-01T00:00:00Z/2024-07-10T23:59:59Z\"),\n", + " # We know that the HLS collection dataset is stored as File per Band\n", + " # so we need to pass a `band_regex` option to assign `bands` to each URL\n", + " (\"bands_regex\", \"B[0-9][0-9]\"),\n", + " # titiler-cmr can work with both Zarr and COG dataset\n", + " # but we need to tell the endpoints in advance which backend\n", + " # to use\n", + " (\"backend\", \"rasterio\"),\n", + " # NDVI\n", + " (\"expression\", \"(B05-B04)/(B05+B04)\"),\n", + " # Need red (B04) and nir (B05) for NDVI\n", + " (\"bands\", \"B05\"),\n", + " (\"bands\", \"B04\"),\n", + " ),\n", + " json=geojson,\n", + " timeout=30,\n", + ").json()\n", + "\n", + "print(json.dumps(r, indent=2))" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/docs/src/examples/SST_Example.ipynb b/docs/src/examples/SST_Example.ipynb new file mode 100644 index 0000000..db81c61 --- /dev/null +++ b/docs/src/examples/SST_Example.ipynb @@ -0,0 +1,307 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cdeef4c6-75b0-44d9-90d4-c850b5d8908a", + "metadata": {}, + "source": [ + "# TiTiler-CMR: Sea Surface Temperature Example\n", + "\n", + "The MUR SST dataset has daily records for sea surface temperature and ice cover fraction. There is a netcdf file for each record.\n", + "\n", + "To run the titiler-cmr service locally you can fire up the docker network with this command:\n", + "```bash\n", + "docker compose up\n", + "```\n", + "\n", + "#### Requirements\n", + "To run some of the chunks in this notebook you will need to install a few packages:\n", + "- `earthaccess`\n", + "- `folium`\n", + "- `httpx`\n", + "- `xarray`\n", + "\n", + "`!pip install folium httpx earthaccess xarray`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7d015182-5347-437a-8b66-d7d62212f0e3", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "from datetime import datetime, timezone\n", + "\n", + "import earthaccess\n", + "import httpx\n", + "import xarray as xr\n", + "from folium import GeoJson, Map, TileLayer\n", + "\n", + "# if running titiler-cmr in the docker network\n", + "titiler_endpoint = \"http://localhost:8081\"" + ] + }, + { + "cell_type": "markdown", + "id": "d375b5b7-9322-4f1e-8859-000ef8ac4898", + "metadata": {}, + "source": [ + "## Identify the dataset\n", + "\n", + "You can find the MUR SST dataset using the `earthaccess.search_datasets` function." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3fcc9cd-6105-42fc-98bf-2de40910a79c", + "metadata": {}, + "outputs": [], + "source": [ + "datasets = earthaccess.search_datasets(doi=\"10.5067/GHGMR-4FJ04\")\n", + "ds = datasets[0]\n", + "\n", + "concept_id = ds[\"meta\"][\"concept-id\"]\n", + "print(\"Concept-Id: \", concept_id)\n", + "\n", + "print(\"Abstract: \", ds[\"umm\"][\"Abstract\"])" + ] + }, + { + "cell_type": "markdown", + "id": "2a4cffa6-0059-4033-a708-db60d743f0e3", + "metadata": {}, + "source": [ + "## Examine a granule\n", + "\n", + "Each granule contains a single day record for the entire globe and has a single data file. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1bde609a-26df-4f35-b7e1-9e1922e87808", + "metadata": {}, + "outputs": [], + "source": [ + "results = earthaccess.search_data(\n", + " count=1,\n", + " concept_id=concept_id,\n", + " temporal=(\"2024-10-12\", \"2024-10-13\"),\n", + ")\n", + "print(\"Granules:\")\n", + "print(results)\n", + "print()\n", + "print(\"Example of NetCDF URL: \")\n", + "for link in results[0].data_links(access=\"external\"):\n", + " print(link)" + ] + }, + { + "cell_type": "markdown", + "id": "eaa3f378-95fa-4c5a-9ccb-24b3064fb5a7", + "metadata": {}, + "source": [ + "## Explore the available variables\n", + "\n", + "The NetCDF file can be opened with xarray using the `h5netcdf` engine. When running outside of AWS region us-west-2 you will need to access the data using \"external\" `https` links (rather than \"direct\" `s3` links). Those links will require authentication which is handled by `earthaccess` as long as you have your Earthdata credentials stored in the `~/.netrc` file!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "61ec4071-bf37-421f-bf58-ac399f827052", + "metadata": {}, + "outputs": [], + "source": [ + "fs = earthaccess.get_fsspec_https_session()\n", + "\n", + "ds = xr.open_dataset(fs.open(results[0].data_links(access=\"external\")[0]), engine=\"h5netcdf\")\n", + "print(\"Data Variables:\")\n", + "for var in ds.data_vars:\n", + " print(str(var))\n", + "\n", + "ds" + ] + }, + { + "cell_type": "markdown", + "id": "885357ee-af17-4a07-a72e-b2aa7ce6cbed", + "metadata": {}, + "source": [ + "## Define a query for titiler-cmr\n", + "\n", + "To use titiler-cmr's endpoints for a NetCDF dataset like this we need to define a date range for the CMR query and a `variable` to analyze." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d13fbdbc-780c-469f-ad1e-14d622e3abc4", + "metadata": {}, + "outputs": [], + "source": [ + "variable = \"sea_ice_fraction\"\n", + "datetime_range = \"/\".join(\n", + " dt.isoformat() for dt in [datetime(2024, 10, 10, tzinfo=timezone.utc), datetime(2024, 10, 11, tzinfo=timezone.utc)]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "bfe85de5-b1ed-4561-802b-d0bea58da1cf", + "metadata": {}, + "source": [ + "## Display tiles in an interactive map\n", + "\n", + "The `/tilejson.json` endpoint will provide a parameterized `xyz` tile URL that can be added to an interactive map." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "00116c09-f16f-4246-8e52-00ef64579abb", + "metadata": {}, + "outputs": [], + "source": [ + "r = httpx.get(\n", + " f\"{titiler_endpoint}/WebMercatorQuad/tilejson.json\",\n", + " params = (\n", + " (\"concept_id\", concept_id),\n", + " # Datetime in form of `start_date/end_date`\n", + " (\"datetime\", datetime_range),\n", + " # titiler-cmr can work with both Zarr and COG dataset\n", + " # but we need to tell the endpoints in advance which backend\n", + " # to use\n", + " (\"backend\", \"xarray\"),\n", + " (\"variable\", variable),\n", + " # We need to set min/max zoom because we don't want to use lowerzoom level (e.g 0)\n", + " # which will results in useless large scale query\n", + " (\"minzoom\", 5),\n", + " (\"maxzoom\", 13),\n", + " (\"rescale\", \"0,1\"),\n", + " (\"colormap_name\", \"blues_r\"),\n", + " )\n", + ").json()\n", + "\n", + "print(r)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "328f9b97-7067-43d1-a918-62bb6470499c", + "metadata": {}, + "outputs": [], + "source": [ + "bounds = r[\"bounds\"]\n", + "m = Map(\n", + " location=(80, -40),\n", + " zoom_start=3\n", + ")\n", + "\n", + "TileLayer(\n", + " tiles=r[\"tiles\"][0],\n", + " opacity=1,\n", + " attr=\"NASA\",\n", + ").add_to(m)\n", + "m" + ] + }, + { + "cell_type": "markdown", + "id": "b2a35be0-b281-4a30-82da-635eadf6d94e", + "metadata": {}, + "source": [ + "## GeoJSON Statistics\n", + "The `/statistics` endpoint can be used to get summary statistics for a geojson `Feature` or `FeatureCollection`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "06d7ef8e-9410-4b35-9d05-e55b4e4972ba", + "metadata": {}, + "outputs": [], + "source": [ + "geojson_dict = {\n", + " \"type\": \"FeatureCollection\",\n", + " \"features\": [\n", + " {\n", + " \"type\": \"Feature\",\n", + " \"properties\": {},\n", + " \"geometry\": {\n", + " \"coordinates\": [\n", + " [\n", + " [\n", + " -20.79973248834736,\n", + " 83.55979308678764\n", + " ],\n", + " [\n", + " -20.79973248834736,\n", + " 75.0115425216471\n", + " ],\n", + " [\n", + " 14.483337068956956,\n", + " 75.0115425216471\n", + " ],\n", + " [\n", + " 14.483337068956956,\n", + " 83.55979308678764\n", + " ],\n", + " [\n", + " -20.79973248834736,\n", + " 83.55979308678764\n", + " ]\n", + " ]\n", + " ],\n", + " \"type\": \"Polygon\"\n", + " }\n", + " }\n", + " ]\n", + "}\n", + "\n", + "r = httpx.post(\n", + " f\"{titiler_endpoint}/statistics\",\n", + " params=(\n", + " (\"concept_id\", concept_id),\n", + " # Datetime in form of `start_date/end_date`\n", + " (\"datetime\", datetime_range),\n", + " # titiler-cmr can work with both Zarr and COG dataset\n", + " # but we need to tell the endpoints in advance which backend\n", + " # to use\n", + " (\"backend\", \"xarray\"),\n", + " (\"variable\", variable),\n", + " ),\n", + " json=geojson_dict,\n", + " timeout=60,\n", + ").json()\n", + "\n", + "print(json.dumps(r, indent=2))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyproject.toml b/pyproject.toml index b917038..64816b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ dependencies = [ "rioxarray~=0.13.4", "s3fs~=2024.9.0", "xarray~=2024.9.0", + "geojson-pydantic>=1.1.1", ] dynamic = ["version"] diff --git a/titiler/cmr/backend.py b/titiler/cmr/backend.py index e4fda53..50307e7 100644 --- a/titiler/cmr/backend.py +++ b/titiler/cmr/backend.py @@ -16,7 +16,8 @@ from earthaccess.auth import Auth from morecantile import Tile, TileMatrixSet from rasterio.crs import CRS -from rasterio.warp import transform_bounds +from rasterio.features import bounds +from rasterio.warp import transform_bounds, transform_geom from rio_tiler.constants import WEB_MERCATOR_TMS, WGS84_CRS from rio_tiler.io import BaseReader, Reader from rio_tiler.models import ImageData @@ -73,7 +74,7 @@ class CMRBackend(BaseBackend): input: str = attr.ib("CMR", init=False) - _backend_name = "CMR" + _backend_name: str = attr.ib(default="CMR") def __attrs_post_init__(self) -> None: """Post Init.""" @@ -315,9 +316,80 @@ def feature( cmr_query: Dict, dst_crs: Optional[CRS] = None, shape_crs: CRS = WGS84_CRS, - max_size: int = 1024, bands_regex: Optional[str] = None, **kwargs: Any, ) -> Tuple[ImageData, List[str]]: """Create an Image from multiple items for a GeoJSON feature.""" - raise NotImplementedError + if "geometry" in shape: + shape = shape["geometry"] + + shape_wgs84 = shape + + if shape_crs != WGS84_CRS: + shape_wgs84 = transform_geom(shape_crs, WGS84_CRS, shape["geometry"]) + + shape_bounds = bounds(shape_wgs84) + + mosaic_assets = self.get_assets( + *shape_bounds, + access=s3_auth_config.access, + bands_regex=bands_regex, + **cmr_query, + ) + + if not mosaic_assets: + raise NoAssetFoundError("No assets found for Geometry") + + def _reader(asset: Asset, shape: Dict, **kwargs: Any) -> ImageData: + if ( + s3_auth_config.strategy == "environment" + and s3_auth_config.access == "direct" + and self.auth + ): + s3_credentials = aws_s3_credential(self.auth, asset["provider"]) + + else: + s3_credentials = None + + if isinstance(self.reader, Reader): + aws_session = None + if s3_credentials: + aws_session = rasterio.session.AWSSession( + aws_access_key_id=s3_credentials["accessKeyId"], + aws_secret_access_key=s3_credentials["secretAccessKey"], + aws_session_token=s3_credentials["sessionToken"], + ) + + with rasterio.Env(aws_session): + with self.reader( + asset["url"], + **self.reader_options, + ) as src_dst: + return src_dst.feature(shape, **kwargs) + + if s3_credentials: + options = { + **self.reader_options, + "s3_credentials": { + "key": s3_credentials["accessKeyId"], + "secret": s3_credentials["secretAccessKey"], + "token": s3_credentials["sessionToken"], + }, + } + else: + options = self.reader_options + + with self.reader( + asset["url"], + **options, + ) as src_dst: + return src_dst.feature(shape, **kwargs) + + return mosaic_reader( + mosaic_assets, + _reader, + shape, + shape_crs=shape_crs, + dst_crs=dst_crs or shape_crs, + **kwargs, + ) diff --git a/titiler/cmr/dependencies.py b/titiler/cmr/dependencies.py index a5219b8..dd32663 100644 --- a/titiler/cmr/dependencies.py +++ b/titiler/cmr/dependencies.py @@ -1,15 +1,18 @@ """titiler-cmr dependencies.""" import datetime as python_datetime -from typing import Any, Dict, List, Literal, Optional, get_args +from dataclasses import dataclass +from typing import Any, Dict, List, Literal, Optional, Union, get_args from ciso8601 import parse_rfc3339 from fastapi import Query +from rio_tiler.types import RIOResampling, WarpResampling from starlette.requests import Request from typing_extensions import Annotated from titiler.cmr.enums import MediaType from titiler.cmr.errors import InvalidDatetime +from titiler.core.dependencies import DefaultDependency ResponseType = Literal["json", "html"] @@ -141,3 +144,100 @@ def cmr_query( raise InvalidDatetime("Invalid datetime: {datetime}") return query + + +@dataclass +class RasterioParams(DefaultDependency): + """Rasterio backend parameters""" + + indexes: Annotated[ + Optional[List[int]], + Query( + title="Band indexes", + alias="bidx", + description="Dataset band indexes", + ), + ] = None + expression: Annotated[ + Optional[str], + Query( + title="Band Math expression", + description="rio-tiler's band math expression", + ), + ] = None + bands: Annotated[ + Optional[List[str]], + Query( + title="Band names", + description="Band names.", + ), + ] = None + bands_regex: Annotated[ + Optional[str], + Query( + title="Regex expression to parse dataset links", + description="Regex expression to parse dataset links.", + ), + ] = None + unscale: Annotated[ + Optional[bool], + Query( + title="Apply internal Scale/Offset", + description="Apply internal Scale/Offset. Defaults to `False`.", + ), + ] = None + resampling_method: Annotated[ + Optional[RIOResampling], + Query( + alias="resampling", + description="RasterIO resampling algorithm. Defaults to `nearest`.", + ), + ] = None + + +@dataclass +class ZarrParams(DefaultDependency): + """Zarr backend parameters""" + + variable: Annotated[ + Optional[str], + Query(description="Xarray Variable"), + ] = None + drop_dim: Annotated[ + Optional[str], + Query(description="Dimension to drop"), + ] = None + time_slice: Annotated[ + Optional[str], Query(description="Slice of time to read (if available)") + ] = None + decode_times: Annotated[ + Optional[bool], + Query( + title="decode_times", + description="Whether to decode times", + ), + ] = None + + +@dataclass +class ReaderParams(DefaultDependency): + """Reader parameters""" + + backend: Annotated[ + Literal["rasterio", "xarray"], + Query(description="Backend to read the CMR dataset"), + ] = "rasterio" + nodata: Annotated[ + Optional[Union[str, int, float]], + Query( + title="Nodata value", + description="Overwrite internal Nodata value", + ), + ] = None + reproject_method: Annotated[ + Optional[WarpResampling], + Query( + alias="reproject", + description="WarpKernel resampling algorithm (only used when doing re-projection). Defaults to `nearest`.", + ), + ] = None diff --git a/titiler/cmr/factory.py b/titiler/cmr/factory.py index 14b314a..e10d3af 100644 --- a/titiler/cmr/factory.py +++ b/titiler/cmr/factory.py @@ -1,21 +1,23 @@ """titiler.cmr.factory: router factories.""" import json +import os import re from dataclasses import dataclass, field -from typing import Any, Dict, List, Literal, Optional, Type, Union +from typing import Any, Dict, Literal, Optional, Tuple, Type, Union from urllib.parse import urlencode import jinja2 import numpy import orjson -from fastapi import APIRouter, Depends, Path, Query +from fastapi import Body, Depends, Path, Query from fastapi.responses import ORJSONResponse -from morecantile import tms as default_tms +from geojson_pydantic import Feature, FeatureCollection from morecantile.defaults import TileMatrixSets +from morecantile.defaults import tms as default_tms from pydantic import conint -from rio_tiler.io import BaseReader, Reader -from rio_tiler.types import RIOResampling, WarpResampling +from rio_tiler.constants import MAX_THREADS, WGS84_CRS +from rio_tiler.io import BaseReader, rasterio from starlette.requests import Request from starlette.responses import HTMLResponse, Response from starlette.routing import compile_path, replace_params @@ -24,14 +26,29 @@ from titiler.cmr import models from titiler.cmr.backend import CMRBackend -from titiler.cmr.dependencies import OutputType, cmr_query +from titiler.cmr.dependencies import ( + OutputType, + RasterioParams, + ReaderParams, + ZarrParams, + cmr_query, +) from titiler.cmr.enums import MediaType from titiler.cmr.reader import MultiFilesBandsReader, ZarrReader -from titiler.core import dependencies from titiler.core.algorithm import algorithms as available_algorithms -from titiler.core.factory import img_endpoint_params +from titiler.core.dependencies import ( + CoordCRSParams, + DefaultDependency, + DstCRSParams, + HistogramParams, + PartFeatureParams, + StatisticsParams, +) +from titiler.core.factory import BaseTilerFactory, img_endpoint_params from titiler.core.models.mapbox import TileJSON +from titiler.core.models.responses import MultiBaseStatisticsGeoJSON from titiler.core.resources.enums import ImageType +from titiler.core.resources.responses import GeoJSONResponse from titiler.core.utils import render_image jinja2_env = jinja2.Environment( @@ -39,6 +56,8 @@ ) DEFAULT_TEMPLATES = Jinja2Templates(env=jinja2_env) +MOSAIC_THREADS = int(os.getenv("MOSAIC_CONCURRENCY", MAX_THREADS)) + def create_html_response( request: Request, @@ -86,41 +105,84 @@ def create_html_response( ) +def parse_reader_options( + rasterio_params: RasterioParams, + zarr_params: ZarrParams, + reader_params: ReaderParams, +) -> Tuple[Type[BaseReader], Dict[str, Any], Dict[str, Any]]: + """Convert rasterio and zarr parameters into a reader and a set of reader_options and read_options""" + + read_options: Dict[str, Any] + reader_options: Dict[str, Any] + options: Dict[str, Any] + reader: Type[BaseReader] + + resampling_method = rasterio_params.resampling_method or "nearest" + + if reader_params.backend == "xarray": + reader = ZarrReader + read_options = {} + + options = { + "variable": zarr_params.variable, + "decode_times": zarr_params.decode_times, + "drop_dim": zarr_params.drop_dim, + "time_slice": zarr_params.time_slice, + } + reader_options = {k: v for k, v in options.items() if v is not None} + else: + if rasterio_params.bands_regex: + assert ( + rasterio_params.bands + ), "`bands=` option must be provided when using Multi bands collections." + + reader = MultiFilesBandsReader + options = { + "expression": rasterio_params.expression, + "bands": rasterio_params.bands, + "unscale": rasterio_params.unscale, + "resampling_method": rasterio_params.resampling_method, + "bands_regex": rasterio_params.bands_regex, + } + read_options = {k: v for k, v in options.items() if v is not None} + reader_options = {} + + else: + assert ( + rasterio_params.bands + ), "Can't use `bands=` option without `bands_regex`" + + reader = rasterio.Reader + options = { + "indexes": rasterio_params.indexes, + "expression": rasterio_params.expression, + "unscale": rasterio_params.unscale, + "resampling_method": resampling_method, + } + read_options = {k: v for k, v in options.items() if v is not None} + reader_options = {} + + return reader, read_options, reader_options + + @dataclass -class Endpoints: +class Endpoints(BaseTilerFactory): """Endpoints Factory.""" - # FastAPI router - router: APIRouter = field(default_factory=APIRouter) - + reader: Optional[Type[BaseReader]] = field(default=None) # type: ignore supported_tms: TileMatrixSets = default_tms - # Router Prefix is needed to find the path for routes when prefixed - # e.g if you mount the route with `/foo` prefix, set router_prefix to foo - router_prefix: str = "" + zarr_dependency: Type[DefaultDependency] = ZarrParams + rasterio_dependency: Type[DefaultDependency] = RasterioParams + reader_dependency: Type[DefaultDependency] = ReaderParams + stats_dependency: Type[DefaultDependency] = StatisticsParams + histogram_dependency: Type[DefaultDependency] = HistogramParams + img_part_dependency: Type[DefaultDependency] = PartFeatureParams templates: Jinja2Templates = DEFAULT_TEMPLATES title: str = "TiTiler-CMR" - def url_for(self, request: Request, name: str, **path_params: Any) -> str: - """Return full url (with prefix) for a specific handler.""" - url_path = self.router.url_path_for(name, **path_params) - - base_url = str(request.base_url) - if self.router_prefix: - prefix = self.router_prefix.lstrip("/") - # If we have prefix with custom path param we check and replace them with - # the path params provided - if "{" in prefix: - _, path_format, param_convertors = compile_path(prefix) - prefix, _ = replace_params( - path_format, param_convertors, request.path_params.copy() - ) - base_url += prefix - - return str(url_path.make_absolute_url(base_url=base_url)) - def _create_html_response( self, request: Request, @@ -135,7 +197,7 @@ def _create_html_response( router_prefix=self.router_prefix, ) - def __post_init__(self): + def register_routes(self): """Post Init: register routes.""" self.register_landing() @@ -143,6 +205,7 @@ def __post_init__(self): self.register_tilematrixsets() self.register_tiles() self.register_map() + self.register_statistics() def register_landing(self) -> None: """register landing page endpoint.""" @@ -418,163 +481,38 @@ def tiles_endpoint( conint(gt=0, le=4), "Tile size scale. 1=256x256, 2=512x512..." ] = 1, format: Annotated[ - ImageType, + Optional[ImageType], "Default will be automatically defined if the output image needs a mask (png) or not (jpeg).", ] = None, - ################################################################### - # CMR options query=Depends(cmr_query), - ################################################################### - backend: Annotated[ - Literal["rasterio", "xarray"], - Query(description="Backend to read the CMR dataset"), - ] = "rasterio", - ################################################################### - # ZarrReader Options - ################################################################### - variable: Annotated[ - Optional[str], - Query(description="Xarray Variable"), - ] = None, - drop_dim: Annotated[ - Optional[str], - Query(description="Dimension to drop"), - ] = None, - time_slice: Annotated[ - Optional[str], Query(description="Slice of time to read (if available)") - ] = None, - decode_times: Annotated[ - Optional[bool], - Query( - title="decode_times", - description="Whether to decode times", - ), - ] = None, - ################################################################### - # Rasterio Reader Options - ################################################################### - indexes: Annotated[ - Optional[List[int]], - Query( - title="Band indexes", - alias="bidx", - description="Dataset band indexes", - ), - ] = None, - expression: Annotated[ - Optional[str], - Query( - title="Band Math expression", - description="rio-tiler's band math expression", - ), - ] = None, - bands: Annotated[ - Optional[List[str]], - Query( - title="Band names", - description="Band names.", - ), - ] = None, - bands_regex: Annotated[ - Optional[str], - Query( - title="Regex expression to parse dataset links", - description="Regex expression to parse dataset links.", - ), - ] = None, - unscale: Annotated[ - Optional[bool], - Query( - title="Apply internal Scale/Offset", - description="Apply internal Scale/Offset. Defaults to `False`.", - ), - ] = None, - resampling_method: Annotated[ - Optional[RIOResampling], - Query( - alias="resampling", - description="RasterIO resampling algorithm. Defaults to `nearest`.", - ), - ] = None, - ################################################################### - # Reader options - ################################################################### - nodata: Annotated[ - Optional[Union[str, int, float]], - Query( - title="Nodata value", - description="Overwrite internal Nodata value", - ), - ] = None, - reproject_method: Annotated[ - Optional[WarpResampling], - Query( - alias="reproject", - description="WarpKernel resampling algorithm (only used when doing re-projection). Defaults to `nearest`.", - ), - ] = None, - ################################################################### - # Rendering Options - ################################################################### - post_process=Depends(available_algorithms.dependency), - rescale=Depends(dependencies.RescalingParams), - color_formula=Depends(dependencies.ColorFormulaParams), - colormap=Depends(dependencies.ColorMapParams), - render_params=Depends(dependencies.ImageRenderingParams), + zarr_params=Depends(self.zarr_dependency), + rasterio_params=Depends(self.rasterio_dependency), + reader_params=Depends(self.reader_dependency), + post_process=Depends(self.process_dependency), + rescale=Depends(self.rescale_dependency), + color_formula=Depends(self.color_formula_dependency), + colormap=Depends(self.colormap_dependency), + render_params=Depends(self.render_dependency), ) -> Response: """Create map tile from a dataset.""" - resampling_method = resampling_method or "nearest" - reproject_method = reproject_method or "nearest" - if nodata is not None: - nodata = numpy.nan if nodata == "nan" else float(nodata) + reproject_method = reader_params.reproject_method or "nearest" + nodata = ( + ( + numpy.nan + if reader_params.nodata == "nan" + else float(reader_params.nodata) + ) + if reader_params.nodata + else None + ) tms = self.supported_tms.get(tileMatrixSetId) - read_options: Dict[str, Any] - reader_options: Dict[str, Any] - options: Dict[str, Any] - reader: Type[BaseReader] - - if backend != "rasterio": - reader = ZarrReader - read_options = {} - - options = { - "variable": variable, - "decode_times": decode_times, - "drop_dim": drop_dim, - "time_slice": time_slice, - } - reader_options = {k: v for k, v in options.items() if v is not None} - else: - if bands_regex: - assert ( - bands - ), "`bands=` option must be provided when using Multi bands collections." - - reader = MultiFilesBandsReader - options = { - "expression": expression, - "bands": bands, - "unscale": unscale, - "resampling_method": resampling_method, - "bands_regex": bands_regex, - } - read_options = {k: v for k, v in options.items() if v is not None} - reader_options = {} - - else: - assert bands, "Can't use `bands=` option without `bands_regex`" - - reader = Reader - options = { - "indexes": indexes, - "expression": expression, - "unscale": unscale, - "resampling_method": resampling_method, - } - read_options = {k: v for k, v in options.items() if v is not None} - reader_options = {} + reader, read_options, reader_options = parse_reader_options( + rasterio_params=rasterio_params, + zarr_params=zarr_params, + reader_params=reader_params, + ) with CMRBackend( tms=tms, @@ -644,92 +582,15 @@ def tilejson_endpoint( # type: ignore Optional[int], Query(description="Overwrite default maxzoom."), ] = None, - ################################################################### - # CMR options query=Depends(cmr_query), - ################################################################### - backend: Annotated[ - Literal["rasterio", "xarray"], - Query(description="Backend to read the CMR dataset"), - ] = "rasterio", - ################################################################### - # ZarrReader Options - ################################################################### - variable: Annotated[ - Optional[str], - Query(description="Xarray Variable"), - ] = None, - drop_dim: Annotated[ - Optional[str], - Query(description="Dimension to drop"), - ] = None, - time_slice: Annotated[ - Optional[str], Query(description="Slice of time to read (if available)") - ] = None, - decode_times: Annotated[ - Optional[bool], - Query( - title="decode_times", - description="Whether to decode times", - ), - ] = None, - ################################################################### - # COG Reader Options - ################################################################### - indexes: Annotated[ - Optional[List[int]], - Query( - title="Band indexes", - alias="bidx", - description="Dataset band indexes", - ), - ] = None, - expression: Annotated[ - Optional[str], - Query( - title="Band Math expression", - description="rio-tiler's band math expression", - ), - ] = None, - unscale: Annotated[ - Optional[bool], - Query( - title="Apply internal Scale/Offset", - description="Apply internal Scale/Offset. Defaults to `False`.", - ), - ] = None, - resampling_method: Annotated[ - Optional[RIOResampling], - Query( - alias="resampling", - description="RasterIO resampling algorithm. Defaults to `nearest`.", - ), - ] = None, - ################################################################### - # Reader options - ################################################################### - nodata: Annotated[ - Optional[Union[str, int, float]], - Query( - title="Nodata value", - description="Overwrite internal Nodata value", - ), - ] = None, - reproject_method: Annotated[ - Optional[WarpResampling], - Query( - alias="reproject", - description="WarpKernel resampling algorithm (only used when doing re-projection). Defaults to `nearest`.", - ), - ] = None, - ################################################################### - # Rendering Options - ################################################################### + zarr_params=Depends(self.zarr_dependency), + rasterio_params=Depends(self.rasterio_dependency), + reader_params=Depends(self.reader_dependency), post_process=Depends(available_algorithms.dependency), - rescale=Depends(dependencies.RescalingParams), - color_formula=Depends(dependencies.ColorFormulaParams), - colormap=Depends(dependencies.ColorMapParams), - render_params=Depends(dependencies.ImageRenderingParams), + rescale=Depends(self.rescale_dependency), + color_formula=Depends(self.color_formula_dependency), + colormap=Depends(self.colormap_dependency), + render_params=Depends(self.render_dependency), ) -> Dict: """Return TileJSON document for a dataset.""" route_params = { @@ -801,6 +662,18 @@ def map_endpoint( # type: ignore Optional[int], Query(description="Overwrite default maxzoom."), ] = None, + query=Depends(cmr_query), + zarr_params=Depends(self.zarr_dependency), + rasterio_params=Depends(self.rasterio_dependency), + reader_params=Depends(self.reader_dependency), + ################################################################### + # Rendering Options + ################################################################### + post_process=Depends(self.process_dependency), + rescale=Depends(self.rescale_dependency), + color_formula=Depends(self.color_formula_dependency), + colormap=Depends(self.colormap_dependency), + render_params=Depends(self.render_dependency), ) -> _TemplateResponse: """Return Map document.""" tilejson_url = self.url_for( @@ -840,3 +713,96 @@ def map_endpoint( # type: ignore }, media_type="text/html", ) + + def register_statistics(self): + """Register /statistics endpoint.""" + + @self.router.post( + "/statistics", + response_model=MultiBaseStatisticsGeoJSON, + response_model_exclude_none=True, + response_class=GeoJSONResponse, + responses={ + 200: { + "content": {"application/geo+json": {}}, + "description": "Return statistics for geojson features.", + } + }, + tags=["Statistics"], + ) + def geojson_statistics( + request: Request, + geojson: Annotated[ + Union[FeatureCollection, Feature], + Body(description="GeoJSON Feature or FeatureCollection."), + ], + coord_crs=Depends(CoordCRSParams), + dst_crs=Depends(DstCRSParams), + query=Depends(cmr_query), + rasterio_params=Depends(self.rasterio_dependency), + zarr_params=Depends(self.zarr_dependency), + reader_params=Depends(self.reader_dependency), + post_process=Depends(self.process_dependency), + stats_params=Depends(self.stats_dependency), + histogram_params=Depends(self.histogram_dependency), + img_part_params=Depends(self.img_part_dependency), + ): + """Get Statistics from a geojson feature or featureCollection.""" + fc = geojson + if isinstance(fc, Feature): + fc = FeatureCollection(type="FeatureCollection", features=[geojson]) + + reader, read_options, reader_options = parse_reader_options( + rasterio_params=rasterio_params, + zarr_params=zarr_params, + reader_params=reader_params, + ) + + with CMRBackend( + reader=reader, + reader_options=reader_options, + auth=request.app.state.cmr_auth, + ) as src_dst: + for feature in fc: + shape = feature.model_dump(exclude_none=True) + + if reader_params.backend == "rasterio": + read_options.update( + { + "threads": MOSAIC_THREADS, + "align_bounds_with_dataset": True, + } + ) + + read_options.update(img_part_params) + + image, _ = src_dst.feature( + shape, + cmr_query=query, + shape_crs=coord_crs or WGS84_CRS, + dst_crs=dst_crs, + **read_options, + ) + + coverage_array = image.get_coverage_array( + shape, + shape_crs=coord_crs or WGS84_CRS, + ) + + if post_process: + image = post_process(image) + + # set band name for statistics method + if not image.band_names and reader_params.backend == "xarray": + image.band_names = [zarr_params.variable] + + stats = image.statistics( + **stats_params, + hist_options={**histogram_params}, + coverage=coverage_array, + ) + + feature.properties = feature.properties or {} + feature.properties.update({"statistics": stats}) + + return fc.features[0] if isinstance(geojson, Feature) else fc