Update functions to use odc-geo and add additional content to tests (…

…#1039) * Make geobox adding func more generic * Remove deprecated map_shapefiles func and replace with .explore * Use odc-geo geobox in xr_animation * Add geomedian collection option to calculate_indices * Update animation notebook to use geomedian so we can add it to tests * Update tests to run on animation notebook * Fix collection number in segmentation notebook * Update rgb func to use odc-geo * Fix attributes being dropped by xr.where, update deprecation version * Remove assign_crs from rgb (not needed), and add error handling to xr_animation * Use better error * Re-run affected notebooks * Update collection param in ML notebooks
GeoscienceAustralia · Apr 21, 2023 · 398d7f4 · 398d7f4
1 parent 6c28e18
commit 398d7f4
Show file tree

Hide file tree

Showing 15 changed files with 1,843 additions and 959 deletions.
diff --git a/How_to_guides/Analyse_multiple_polygons.ipynb b/How_to_guides/Analyse_multiple_polygons.ipynb
diff --git a/How_to_guides/Animated_timeseries.ipynb b/How_to_guides/Animated_timeseries.ipynb
diff --git a/How_to_guides/Calculating_band_indices.ipynb b/How_to_guides/Calculating_band_indices.ipynb
@@ -327,8 +327,10 @@
    "source": [
     "> **Note**: when using the `calculate_indices` function, it is important to set the `collection` parameter correctly. \n",
     "This is because different satellite collections use different names for the same bands, which can lead to invalid results if not accounted for. \n",
-    "For Landsat (i.e. GA Landsat Collection 3), specify `collection='ga_ls_3'`. \n",
-    "For Sentinel 2 (i.e. GA Sentinel 2 Collection 1), specify `collection='ga_s2_1'`."
+    "\n",
+    "> * For Landsat (i.e. GA Landsat Collection 3), specify `collection='ga_ls_3'`. \n",
+    "> * For Sentinel 2 (i.e. GA Sentinel 2 Collection 3), specify `collection='ga_s2_3'`.\n",
+    "> * For Landsat geomedian products, specify `collection='ga_gm_3'`."
    ]
   },
   {

diff --git a/How_to_guides/Generating_composites.ipynb b/How_to_guides/Generating_composites.ipynb
diff --git a/How_to_guides/Image_segmentation.ipynb b/How_to_guides/Image_segmentation.ipynb
diff --git a/How_to_guides/Principal_component_analysis.ipynb b/How_to_guides/Principal_component_analysis.ipynb
@@ -357,7 +357,7 @@
     "**Contact:** If you need assistance, please post a question on the [Open Data Cube Slack channel](http://slack.opendatacube.org/) or on the [GIS Stack Exchange](https://gis.stackexchange.com/questions/ask?tags=open-data-cube) using the `open-data-cube` tag (you can view previously asked questions [here](https://gis.stackexchange.com/questions/tagged/open-data-cube)).\n",
     "If you would like to report an issue with this notebook, you can file one on [Github](https://github.com/GeoscienceAustralia/dea-notebooks).\n",
     "\n",
-    "**Last modified:** February 2023\n",
+    "**Last modified:** April 2023\n",
     "\n",
     "**Compatible datacube version:** "
    ]
@@ -371,7 +371,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "1.8.11\n"
+      "1.8.12\n"
      ]
     }
    ],

diff --git a/Real_world_examples/Scalable_machine_learning/1_Extract_training_data.ipynb b/Real_world_examples/Scalable_machine_learning/1_Extract_training_data.ipynb
@@ -331,7 +331,7 @@
     "    da = calculate_indices(ds,\n",
     "                           index=['NDVI', 'LAI', 'MNDWI'],\n",
     "                           drop=False,\n",
-    "                           collection='ga_ls_2')\n",
+    "                           collection='ga_gm_3')\n",
     "    \n",
     "    # Add Fractional cover percentiles\n",
     "    fc = dc.load(product='ga_ls_fc_pc_cyear_3',\n",
@@ -550,97 +550,101 @@
   "widgets": {
    "application/vnd.jupyter.widget-state+json": {
     "state": {
-     "2011e37b55a741df950da780f28638a6": {
-      "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "1.5.0",
-      "model_name": "HTMLModel",
-      "state": {
-       "layout": "IPY_MODEL_851e04ec60314ff885be1d8687ac3fe9",
-       "style": "IPY_MODEL_c23cfc374b1945e9966e903069e7aaa8",
-       "value": " 430/430 [08:35&lt;00:00,  1.44s/it]"
-      }
+     "3c2d1d7f832a4fe7b7797e22e2ebdbcb": {
+      "model_module": "@jupyter-widgets/base",
+      "model_module_version": "2.0.0",
+      "model_name": "LayoutModel",
+      "state": {}
      },
-     "27b7c4fb86f741b780ba57e2bcfeefbe": {
+     "4baac50e5a3444b0a6eb20ecb839fd63": {
       "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "1.5.0",
+      "model_module_version": "2.0.0",
       "model_name": "FloatProgressModel",
       "state": {
        "bar_style": "success",
-       "layout": "IPY_MODEL_3c207694fca34d89a8d87d58f91e3012",
+       "layout": "IPY_MODEL_3c2d1d7f832a4fe7b7797e22e2ebdbcb",
        "max": 430,
-       "style": "IPY_MODEL_cf3f69b6c916458aa75c9b0738c9c1a4",
+       "style": "IPY_MODEL_f3d1b067aea84dc7b6b5798394f171b2",
        "value": 430
       }
      },
-     "3c207694fca34d89a8d87d58f91e3012": {
+     "607cbb4bd78f4468981e715eb5e72d67": {
       "model_module": "@jupyter-widgets/base",
-      "model_module_version": "1.2.0",
+      "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
       "state": {}
      },
-     "450d0fa0c82e460f84364477a6de7468": {
-      "model_module": "@jupyter-widgets/base",
-      "model_module_version": "1.2.0",
-      "model_name": "LayoutModel",
-      "state": {}
+     "611d1ba2720f41d8801715cf7b730ed5": {
+      "model_module": "@jupyter-widgets/controls",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
+      "state": {
+       "layout": "IPY_MODEL_b0fa4da58bdd4964b89dbb6332a77ed6",
+       "style": "IPY_MODEL_d659821494a14b1cbd71af4f25c4903a",
+       "value": " 430/430 [08:47&lt;00:00,  1.68s/it]"
+      }
      },
-     "6551a60ab4f348c59d0ad849fb7c0615": {
+     "86566c2c69b3438583c77b2f76f7910c": {
       "model_module": "@jupyter-widgets/base",
-      "model_module_version": "1.2.0",
+      "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
       "state": {}
      },
-     "851e04ec60314ff885be1d8687ac3fe9": {
+     "b0fa4da58bdd4964b89dbb6332a77ed6": {
       "model_module": "@jupyter-widgets/base",
-      "model_module_version": "1.2.0",
+      "model_module_version": "2.0.0",
       "model_name": "LayoutModel",
       "state": {}
      },
-     "c23cfc374b1945e9966e903069e7aaa8": {
+     "b16dccfabc5c43a68990897ce4f077a1": {
       "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "1.5.0",
-      "model_name": "DescriptionStyleModel",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
       "state": {
-       "description_width": ""
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
       }
      },
-     "c3341c8f347c47d396b0dca98cae061b": {
+     "cdb33a65a8394d23869504dbd98fe6a6": {
       "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "1.5.0",
-      "model_name": "DescriptionStyleModel",
+      "model_module_version": "2.0.0",
+      "model_name": "HBoxModel",
       "state": {
-       "description_width": ""
+       "children": [
+        "IPY_MODEL_e901507e78154c74ab15739b9b75a64f",
+        "IPY_MODEL_4baac50e5a3444b0a6eb20ecb839fd63",
+        "IPY_MODEL_611d1ba2720f41d8801715cf7b730ed5"
+       ],
+       "layout": "IPY_MODEL_607cbb4bd78f4468981e715eb5e72d67"
       }
      },
-     "c73da0bce94b4503be366cdcb4bf25dd": {
+     "d659821494a14b1cbd71af4f25c4903a": {
       "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "1.5.0",
-      "model_name": "HTMLModel",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLStyleModel",
       "state": {
-       "layout": "IPY_MODEL_450d0fa0c82e460f84364477a6de7468",
-       "style": "IPY_MODEL_c3341c8f347c47d396b0dca98cae061b",
-       "value": "100%"
+       "description_width": "",
+       "font_size": null,
+       "text_color": null
       }
      },
-     "cf3f69b6c916458aa75c9b0738c9c1a4": {
+     "e901507e78154c74ab15739b9b75a64f": {
       "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "1.5.0",
-      "model_name": "ProgressStyleModel",
+      "model_module_version": "2.0.0",
+      "model_name": "HTMLModel",
       "state": {
-       "description_width": ""
+       "layout": "IPY_MODEL_86566c2c69b3438583c77b2f76f7910c",
+       "style": "IPY_MODEL_b16dccfabc5c43a68990897ce4f077a1",
+       "value": "100%"
       }
      },
-     "d28cdda5ed0941c4907430eb3165b157": {
+     "f3d1b067aea84dc7b6b5798394f171b2": {
       "model_module": "@jupyter-widgets/controls",
-      "model_module_version": "1.5.0",
-      "model_name": "HBoxModel",
+      "model_module_version": "2.0.0",
+      "model_name": "ProgressStyleModel",
       "state": {
-       "children": [
-        "IPY_MODEL_c73da0bce94b4503be366cdcb4bf25dd",
-        "IPY_MODEL_27b7c4fb86f741b780ba57e2bcfeefbe",
-        "IPY_MODEL_2011e37b55a741df950da780f28638a6"
-       ],
-       "layout": "IPY_MODEL_6551a60ab4f348c59d0ad849fb7c0615"
+       "description_width": ""
       }
      }
     },

diff --git a/Real_world_examples/Scalable_machine_learning/4_Classify_satellite_data.ipynb b/Real_world_examples/Scalable_machine_learning/4_Classify_satellite_data.ipynb
@@ -225,7 +225,7 @@
     "    da = calculate_indices(ds,\n",
     "                           index=['NDVI', 'LAI', 'MNDWI'],\n",
     "                           drop=False,\n",
-    "                           collection='ga_ls_2')\n",
+    "                           collection='ga_gm_3')\n",
     "    \n",
     "    # Add Fractional cover percentiles\n",
     "    fc = dc.load(product='ga_ls_fc_pc_cyear_3',\n",

diff --git a/Supplementary_data/Animated_timeseries/vector.geojson b/Supplementary_data/Animated_timeseries/vector.geojson
diff --git a/Tests/setup_test_datacube.sh b/Tests/setup_test_datacube.sh
@@ -38,6 +38,12 @@ s3-to-dc 's3://dea-public-data/derivative/ga_ls8c_nbart_gm_cyear_3/3-0-0/x49/y24
 s3-to-dc 's3://dea-public-data/derivative/ga_ls8c_nbart_gm_cyear_3/3-0-0/x49/y24/2014--P1Y/*.odc-metadata.yaml' --no-sign-request --skip-lineage 'ga_ls8c_nbart_gm_cyear_3'
 s3-to-dc 's3://dea-public-data/derivative/ga_ls8c_nbart_gm_cyear_3/3-0-0/x49/y24/2015--P1Y/*.odc-metadata.yaml' --no-sign-request --skip-lineage 'ga_ls8c_nbart_gm_cyear_3'
 s3-to-dc 's3://dea-public-data/derivative/ga_ls8c_nbart_gm_cyear_3/3-0-0/x49/y24/2016--P1Y/*.odc-metadata.yaml' --no-sign-request --skip-lineage 'ga_ls8c_nbart_gm_cyear_3'
+s3-to-dc 's3://dea-public-data/derivative/ga_ls8c_nbart_gm_cyear_3/3-0-0/x38/y19/2017--P1Y/*.odc-metadata.yaml' --no-sign-request --skip-lineage 'ga_ls8c_nbart_gm_cyear_3'
+s3-to-dc 's3://dea-public-data/derivative/ga_ls8c_nbart_gm_cyear_3/3-0-0/x38/y19/2018--P1Y/*.odc-metadata.yaml' --no-sign-request --skip-lineage 'ga_ls8c_nbart_gm_cyear_3'
+s3-to-dc 's3://dea-public-data/derivative/ga_ls8c_nbart_gm_cyear_3/3-0-0/x38/y19/2016--P1Y/*.odc-metadata.yaml' --no-sign-request --skip-lineage 'ga_ls8c_nbart_gm_cyear_3'
+s3-to-dc 's3://dea-public-data/derivative/ga_ls8c_nbart_gm_cyear_3/3-0-0/x38/y19/2015--P1Y/*.odc-metadata.yaml' --no-sign-request --skip-lineage 'ga_ls8c_nbart_gm_cyear_3'
+s3-to-dc 's3://dea-public-data/derivative/ga_ls8c_nbart_gm_cyear_3/3-0-0/x38/y19/2019--P1Y/*.odc-metadata.yaml' --no-sign-request --skip-lineage 'ga_ls8c_nbart_gm_cyear_3'
+s3-to-dc 's3://dea-public-data/derivative/ga_ls8c_nbart_gm_cyear_3/3-0-0/x38/y19/2020--P1Y/*.odc-metadata.yaml' --no-sign-request --skip-lineage 'ga_ls8c_nbart_gm_cyear_3'
 
 # Index FC
 s3-to-dc 's3://dea-public-data/derivative/ga_ls_fc_3/2-5-0/096/084/1993/10/30/*.json' --no-sign-request --skip-lineage --stac 'ga_ls_fc_3'

diff --git a/Tests/test_notebooks.sh b/Tests/test_notebooks.sh
@@ -7,6 +7,6 @@ set -o pipefail
 cd ./dea-notebooks
 pip3 install ./Tools
 
-pytest --durations=10 --nbval-lax Beginners_guide DEA_products How_to_guides/Contour_extraction.ipynb How_to_guides/Calculating_band_indices.ipynb How_to_guides/Downloading_data_with_STAC.ipynb How_to_guides/Exporting_GeoTIFFs.ipynb How_to_guides/Generating_composites.ipynb How_to_guides/Image_segmentation.ipynb How_to_guides/Opening_GeoTIFFs_NetCDFs.ipynb How_to_guides/Pansharpening.ipynb How_to_guides/Polygon_drill.ipynb How_to_guides/Principal_component_analysis.ipynb How_to_guides/Rasterize_vectorize.ipynb How_to_guides/Using_load_ard.ipynb How_to_guides/Virtual_products.ipynb
+pytest --durations=10 --nbval-lax Beginners_guide DEA_products How_to_guides/Animated_timeseries.ipynb How_to_guides/Contour_extraction.ipynb How_to_guides/Calculating_band_indices.ipynb How_to_guides/Downloading_data_with_STAC.ipynb How_to_guides/Exporting_GeoTIFFs.ipynb How_to_guides/Generating_composites.ipynb How_to_guides/Image_segmentation.ipynb How_to_guides/Opening_GeoTIFFs_NetCDFs.ipynb How_to_guides/Pansharpening.ipynb How_to_guides/Polygon_drill.ipynb How_to_guides/Principal_component_analysis.ipynb How_to_guides/Rasterize_vectorize.ipynb How_to_guides/Using_load_ard.ipynb How_to_guides/Virtual_products.ipynb
 
 
diff --git a/Tools/dea_tools/bandindices.py b/Tools/dea_tools/bandindices.py
@@ -39,7 +39,7 @@ def calculate_indices(ds,
     in memory. This can be a memory-expensive operation, so to avoid
     this, set `inplace=True`.
 
-    Last modified: March 2021
+    Last modified: April 2023
     
     Parameters
     ----------
@@ -98,10 +98,9 @@ def calculate_indices(ds,
         
         Valid options are: 
         
-        * ``'ga_ls_2'`` (for GA Landsat Collection 2)
         * ``'ga_ls_3'`` (for GA Landsat Collection 3) 
-        * ``'ga_s2_1'`` (for GA Sentinel 2 Collection 1)
         * ``'ga_s2_3'`` (for GA Sentinel 2 Collection 3)
+        * ``'ga_gm_3'`` (for GA Geomedian Collection 3)
 
     custom_varname : str, optional
         By default, the original dataset will be returned with 
@@ -299,7 +298,7 @@ def calculate_indices(ds,
     # iterate through either multiple or single indices in the loop below
     indices = index if isinstance(index, list) else [index]
 
-    #calculate for each index in the list of indices supplied (indexes)
+    # Calculate for each index in the list of indices supplied (indexes)
     for index in indices:
 
         # Select an index function from the dictionary
@@ -338,10 +337,10 @@ def calculate_indices(ds,
         if collection is None:
 
             raise ValueError("'No `collection` was provided. Please specify "
-                             "either 'ga_ls_2', 'ga_ls_3', 'ga_s2_1' or "
-                             "'ga_s2_3' to ensure the function calculates indices using the "
-                             "correct spectral bands")
-
+                             "either 'ga_ls_3', 'ga_s2_3' or 'ga_gm_3' "
+                             "to ensure the function calculates indices "
+                             "using the correct spectral bands")            
+        
         elif collection == 'ga_ls_3':
 
             # Dictionary mapping full data names to simpler 'red' alias names
@@ -365,7 +364,7 @@ def calculate_indices(ds,
                 a: b for a, b in bandnames_dict.items() if a in ds.variables
             }
 
-        elif (collection == 'ga_s2_1') | (collection == 'ga_s2_3'):
+        elif collection == 'ga_s2_3':
 
             # Dictionary mapping full data names to simpler 'red' alias names
             bandnames_dict = {
@@ -391,17 +390,17 @@ def calculate_indices(ds,
             bands_to_rename = {
                 a: b for a, b in bandnames_dict.items() if a in ds.variables
             }
-
-        elif collection == 'ga_ls_2':
-
+           
+        elif collection == 'ga_gm_3':
+            
             # Pass an empty dict as no bands need renaming
             bands_to_rename = {}
 
         # Raise error if no valid collection name is provided:
         else:
             raise ValueError(f"'{collection}' is not a valid option for "
-                              "`collection`. Please specify either \n"
-                              "'ga_ls_2', 'ga_ls_3', 'ga_s2_1' or 'ga_s2_3'")
+                             "`collection`. Please specify either \n"
+                             "'ga_ls_3', 'ga_s2_3' or 'ga_gm_3'")
 
         # Apply index function 
         try:

diff --git a/Tools/dea_tools/datahandling.py b/Tools/dea_tools/datahandling.py
@@ -483,7 +483,7 @@ def load_ard(
         pq_mask = ~mask_cleanup(~pq_mask, mask_filters=mask_filters)
 
         warnings.warn(
-            "As of `dea_tools` v0.3.0, pixel quality masks are "
+            "As of `dea_tools` v1.0.0, pixel quality masks are "
             "inverted before being passed to `mask_filters` (i.e. so "
             "that good quality/clear pixels are False and poor quality "
             "pixels/clouds are True). This means that 'dilation' will "
@@ -887,13 +887,14 @@ def nearest(
 
     target = array[dim].dtype.type(target)
     is_before_closer = abs(target - da_before[dim]) < abs(target - da_after[dim])
-    nearest_array = xr.where(is_before_closer, da_before, da_after)
-    nearest_array[dim] = xr.where(is_before_closer, da_before[dim], da_after[dim])
+    nearest_array = xr.where(is_before_closer, da_before, da_after, keep_attrs=True)
+    nearest_array[dim] = xr.where(is_before_closer, da_before[dim], da_after[dim], keep_attrs=True)
 
     if index_name is not None:
         nearest_array[index_name] = xr.where(
-            is_before_closer, da_before[index_name], da_after[index_name]
+            is_before_closer, da_before[index_name], da_after[index_name], keep_attrs=True
         )
+
     return nearest_array