From 6bd4e49c4acfe27ad28e6d0df6498c420061af3c Mon Sep 17 00:00:00 2001 From: BRAUN REMI Date: Mon, 6 Mar 2023 12:33:21 +0100 Subject: [PATCH] OPTIM: Always use chunks when reading rasters #58 --- CHANGES.md | 6 ++++++ docs/sar.md | 14 +++++++------- eoreader/env_vars.py | 5 +++++ eoreader/utils.py | 9 ++++----- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2a8996cf..87f4e912 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,11 @@ # Release History +## 0.YY.Z (2023-MM-DD) + +### Other + +- OPTIM: Always use chunks when reading rasters ([#58](https://github.com/sertit/eoreader/issues/58)) + ## 0.19.2 (2023-02-23) ### Bug Fixes diff --git a/docs/sar.md b/docs/sar.md index b39120e8..c7ac1dc1 100644 --- a/docs/sar.md +++ b/docs/sar.md @@ -31,16 +31,16 @@ mostly because SNAP doesn't handle them. | `Capella` | SICD, SIDD, CPHD | ❌ | | `COSMO-Skymed` | SCS | ✅ | | `COSMO-SkyMed` 1st Generation | DGM | ✅ | -| `COSMO-SkyMed` 2nd Generation | DGM | ⚠️ | -| `COSMO-SkyMed` | GEC, GTC | ⚠️ | +| `COSMO-SkyMed` 2nd Generation | DGM | ⚠ | +| `COSMO-SkyMed` | GEC, GTC | ⚠ | | `ICEYE` | SLC | ✅ | | `ICEYE` | GRD | ✅ | | `ICEYE` | ORTHO | 💤 | -| `RADARSAT Constellation Mission` | SLC | ⚠️ | -| `RADARSAT Constellation Mission` | GRC, GCC, GCD | ⚠️ | +| `RADARSAT Constellation Mission` | SLC | ⚠ | +| `RADARSAT Constellation Mission` | GRC, GCC, GCD | ⚠ | | `RADARSAT Constellation Mission` | GRD | ✅ | | `RADARSAT-2` | SLC | ✅ | -| `RADARSAT-2` | SGX, SCN, SCW,
SCF, SCS, SSG, SPG | ⚠️ | +| `RADARSAT-2` | SGX, SCN, SCW,
SCF, SCS, SSG, SPG | ⚠ | | `RADARSAT-2` | SGF | ✅ | | `Sentinel-1` | SLC | ✅ | | `Sentinel-1` | GRD | ✅ | @@ -50,11 +50,11 @@ mostly because SNAP doesn't handle them. | `SAOCOM-1` | GTC | ✅ | | `TerraSAR-X`, `TanDEM-X`, `PAZ SAR` | SSC | ✅ | | `TerraSAR-X`, `TanDEM-X`, `PAZ SAR` | MGD | ✅ | -| `TerraSAR-X`, `TanDEM-X`, `PAZ SAR` | GEC | ⚠️ | +| `TerraSAR-X`, `TanDEM-X`, `PAZ SAR` | GEC | ⚠ | | `TerraSAR-X`, `TanDEM-X`, `PAZ SAR` | EEC | ✅ | ✅: Tested -⚠️: Never tested, **use it at your own risk!** +⚠: Never tested, **use it at your own risk!** ❌: Not handled 💤: Waiting for the release diff --git a/eoreader/env_vars.py b/eoreader/env_vars.py index 7365693b..b91aa85b 100644 --- a/eoreader/env_vars.py +++ b/eoreader/env_vars.py @@ -55,3 +55,8 @@ """ If set and :code:`dask` is installed, EOReader will read products as dask arrays instead of numpy arrays. """ + +TILE_SIZE = "EOREADER_TILE_SIZE" +""" +If set, overrides the default tile size used in chunking (2048 by default) +""" diff --git a/eoreader/utils.py b/eoreader/utils.py index 3501fa07..8d5574db 100644 --- a/eoreader/utils.py +++ b/eoreader/utils.py @@ -37,7 +37,7 @@ from eoreader import EOREADER_NAME from eoreader.bands import is_index, is_sat_band, to_str -from eoreader.env_vars import USE_DASK +from eoreader.env_vars import TILE_SIZE, USE_DASK from eoreader.exceptions import InvalidProductError from eoreader.keywords import _prune_keywords @@ -159,10 +159,9 @@ def read( """ window = kwargs.get("window") - if use_dask(): - chunks = True - else: - chunks = None + # Always use chunks + tile_size = os.getenv(TILE_SIZE, 2048) + chunks = [tile_size, tile_size] try: # Disable georef warnings here as the SAR/Sentinel-3 products are not georeferenced