From a7b26f38623ce45d79e4be416d52c286e63e3876 Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Fri, 25 Oct 2024 17:26:38 -0400 Subject: [PATCH 1/3] Update readme --- README.md | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index ea910f8..aa811d7 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,6 @@ ## Overview -This a work-in-progress guidebook on existing warp resampling / reprojection methods in Python, along with some memory and statistical wall-time profiling results. - -## Caveat - -This is a WIP guidebook. We are presenting early on in the development to guide discussions and future work. The output from different resampling methods has not yet been verified, important parameters (e.g., target no_data values) have not been set, and not all methods have been implemented. We encourage people to contribute by building on the notebooks in the `examples` directory or participating in discussions on this repo or on the Pangeo discourse. +This repository and the accompanying Quarto book contains examples of using warp resampling / reprojection methods in Python, along with memory and statistical wall-time profiling results. ## Memory and time profiling @@ -12,7 +8,7 @@ Resampling and reprojection (i.e., warp resampling) are essential steps for gene ### Goals -Compare memory and time performance for generating a zoom level 0 256 x 256 raster from one timestep and variable of the MUR SST dataset using the following approaches: +Compare memory and time performance for generating Web Mercator Quad Tree raster tiles from one timestep and variable of the MUR SST and GPM IMERG dataset using the following approaches: - [osgeo.warp](https://gdal.org/en/latest/api/python/utilities.html#osgeo.gdal.Warp) - [rasterio.warp.reproject](https://rasterio.readthedocs.io/en/stable/api/rasterio.warp.html#rasterio.warp.reproject) @@ -33,14 +29,7 @@ Out-of-scope: - [pygmt.grdproject](https://www.pygmt.org/latest/api/generated/pygmt.grdproject.html#pygmt.grdproject) - [web mercator not amongst supported projections](https://www.pygmt.org/latest/projections/index.html) - [verde](https://www.fatiando.org/verde/latest/) - not used for raster -> raster resampling (only points -> raster) -These methods will be run on the full resolution dataset. Nearest neighbor interpolation will be used for the first comparison. For simplicity, the amount of time necessary to generate a resampled array and the maximum amount of heap memory allocated will be measured. - -### Planned extensions - -- Compare to results when using a 2x and 4x downsampled versions to better understand the time and memory complexity -- Compare to results when using a virtual dataset (e.g., VRT, Kerchunk reference file). -- Compare results when reading from a dataset stored locally versus in cloud object storage. -- Compare to results when using a cloud-optimized dataset (Zarr). +These methods will be run on the full resolution dataset. Nearest neighbor interpolation will be used for the first comparison. For simplicity, the amount of time necessary to generate a resampled array and the maximum amount of heap memory allocated will be measured. We also compare results when using a virtual dataset (e.g., VRT, Kerchunk reference file), when reading from a dataset stored locally versus in cloud object storage, and when using a cloud-optimized dataset (Zarr.) ### Possible extensions From 0f9afad7a70ef76b13c1fd3e4b934d2c088a98ac Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Fri, 25 Oct 2024 17:26:53 -0400 Subject: [PATCH 2/3] Update slides --- pages/slides.qmd | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pages/slides.qmd b/pages/slides.qmd index a149621..3b35762 100644 --- a/pages/slides.qmd +++ b/pages/slides.qmd @@ -96,14 +96,14 @@ Visualization ## {background-iframe="https://developmentseed.org/warp-resample-profiling/examples/earthdata-download.html"} -## {background-iframe="https://developmentseed.org/warp-resample-profiling/examples/resample-gdal-netcdf-vsis3.html"} +## {background-iframe="https://developmentseed.org/warp-resample-profiling/examples/resample-netcdf-rasterio-netcdf-local.html"} -## {background-iframe="https://developmentseed.org/warp-resample-profiling/examples/resample-odc-h5netcdf-.html"} - -## {background-iframe="https://developmentseed.org/warp-resample-profiling/examples/future-resample-xesmf-h5netcdf-.html"} +## {background-iframe="https://developmentseed.org/warp-resample-profiling/examples/generate-weights.html"} ## {background-iframe="https://developmentseed.org/warp-resample-profiling/examples/process-results.html"} +## {background-iframe="https://developmentseed.org/warp-resample-profiling/examples/process-gpm-results.html"} + ## {background-iframe="https://link.excalidraw.com/readonly/GgF1qYswczuP9WKpoCvU"} ## Observations and opinions From 2eb881aa7f9717b97e8fc912f653f588f630fd2e Mon Sep 17 00:00:00 2001 From: Max Jones <14077947+maxrjones@users.noreply.github.com> Date: Fri, 25 Oct 2024 17:28:14 -0400 Subject: [PATCH 3/3] Add notebook title --- examples/generate-weights.ipynb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/examples/generate-weights.ipynb b/examples/generate-weights.ipynb index c470902..ce2925c 100644 --- a/examples/generate-weights.ipynb +++ b/examples/generate-weights.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Pre-generate weights for resampling with XESMF or sparse" + ] + }, { "cell_type": "code", "execution_count": null, @@ -265,7 +272,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.10" + "version": "3.12.6" } }, "nbformat": 4,