From 978f92105bbc200c2d92badc2f78ad9f96d53081 Mon Sep 17 00:00:00 2001
From: Lauren Gallic <lawi@microsoft.com>
Date: Thu, 2 Nov 2023 15:08:11 +0000
Subject: [PATCH] moved constant kerchunk variables to collection asset and
 kept unique ones for the item level

---
 .../_kerchunk_helper_functions.py             | 23 +++++++++++++--
 src/stactools/ecmwf_forecast/stac.py          | 29 +++++++++++++++++++
 2 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/src/stactools/ecmwf_forecast/_kerchunk_helper_functions.py b/src/stactools/ecmwf_forecast/_kerchunk_helper_functions.py
index 40345a7..74aa9a9 100644
--- a/src/stactools/ecmwf_forecast/_kerchunk_helper_functions.py
+++ b/src/stactools/ecmwf_forecast/_kerchunk_helper_functions.py
@@ -71,8 +71,27 @@ def get_kerchunk_indices(part):
     elif ((part.stream == "scwv") or (part.stream == "wave")) and (part.type == "fc"):
         mzz = MultiZarrToZarr(out, concat_dims=["time"])
 
-    return convert_base64(compress_lat_lon(mzz.translate()))
-
+    #get output, filter down and only keep the unique d['refs'] items
+    d = convert_base64(compress_lat_lon(mzz.translate()))
+    new_d = {}
+    new_d['refs'] = dict(filter(filter_kerchunk(d['refs'].items())))
+
+    return new_d
+
+def filter_kerchunk(pair):
+    wanted_keys = ['time/0',
+                   'mp2/0.0.0',
+                   'mwd/0.0.0',
+                   'mwp/0.0.0',
+                   'swh/0.0.0',
+                   'pp1d/0.0.0',
+                   'valid_time/0',
+                   'step/0']
+    key, value = pair
+    if key in wanted_keys:
+        return True
+    else:
+        return False
 
 def convert_base64(d):
     for key in d['refs']:
diff --git a/src/stactools/ecmwf_forecast/stac.py b/src/stactools/ecmwf_forecast/stac.py
index 606452a..3be680e 100644
--- a/src/stactools/ecmwf_forecast/stac.py
+++ b/src/stactools/ecmwf_forecast/stac.py
@@ -211,6 +211,35 @@ def create_collection(
             {
                 "type": GRIB2_MEDIA_TYPE,
                 "roles": ["data"],
+                "kerchunk:indices": {'refs': {'step/0': 'base64:AAAAAAAAXkA=',
+  '.zattrs': '{"GRIB_centre":"ecmf","GRIB_centreDescription":"European Centre for Medium-Range Weather Forecasts","GRIB_edition":2,"GRIB_subCentre":0,"coordinates":"meanSea latitude longitude step time valid_time","institution":"European Centre for Medium-Range Weather Forecasts"}',
+  '.zgroup': '{"zarr_format":2}',
+  'meanSea/0': 'base64:AAAAAAAAAAA=',
+  'latitude/0': 'base64:AAAAAACAVkCamZmZmZlWwJqZmZmZmdm/',
+  'longitude/0': 'base64:AAAAAACAZsAAAAAAAIBmQJqZmZmZmdk/',
+  'mp2/.zarray': '{"chunks":[1,451,900],"compressor":null,"dtype":"<f8","fill_value":null,"filters":[{"dtype":"float64","id":"grib","var":"mp2"}],"order":"C","shape":[1,451,900],"zarr_format":2}',
+  'mp2/.zattrs': '{"GRIB_NV":0,"GRIB_Nx":900,"GRIB_Ny":451,"GRIB_cfName":"unknown","GRIB_cfVarName":"mp2","GRIB_dataType":"fc","GRIB_gridDefinitionDescription":"Latitude\\/longitude","GRIB_gridType":"regular_ll","GRIB_iDirectionIncrementInDegrees":0.4,"GRIB_iScansNegatively":0,"GRIB_jDirectionIncrementInDegrees":0.4,"GRIB_jPointsAreConsecutive":0,"GRIB_jScansPositively":0,"GRIB_latitudeOfFirstGridPointInDegrees":90.0,"GRIB_latitudeOfLastGridPointInDegrees":-90.0,"GRIB_longitudeOfFirstGridPointInDegrees":180.0,"GRIB_longitudeOfLastGridPointInDegrees":179.6,"GRIB_missingValue":3.4028234663852886e+38,"GRIB_name":"Mean zero-crossing wave period","GRIB_numberOfPoints":405900,"GRIB_paramId":140221,"GRIB_shortName":"mp2","GRIB_stepType":"instant","GRIB_stepUnits":1,"GRIB_typeOfLevel":"meanSea","GRIB_units":"s","_ARRAY_DIMENSIONS":["time","latitude","longitude"],"long_name":"Mean zero-crossing wave period","standard_name":"unknown","units":"s"}',
+  'mwd/.zarray': '{"chunks":[1,451,900],"compressor":null,"dtype":"<f8","fill_value":null,"filters":[{"dtype":"float64","id":"grib","var":"mwd"}],"order":"C","shape":[1,451,900],"zarr_format":2}',
+  'mwd/.zattrs': '{"GRIB_NV":0,"GRIB_Nx":900,"GRIB_Ny":451,"GRIB_cfName":"unknown","GRIB_cfVarName":"mwd","GRIB_dataType":"fc","GRIB_gridDefinitionDescription":"Latitude\\/longitude","GRIB_gridType":"regular_ll","GRIB_iDirectionIncrementInDegrees":0.4,"GRIB_iScansNegatively":0,"GRIB_jDirectionIncrementInDegrees":0.4,"GRIB_jPointsAreConsecutive":0,"GRIB_jScansPositively":0,"GRIB_latitudeOfFirstGridPointInDegrees":90.0,"GRIB_latitudeOfLastGridPointInDegrees":-90.0,"GRIB_longitudeOfFirstGridPointInDegrees":180.0,"GRIB_longitudeOfLastGridPointInDegrees":179.6,"GRIB_missingValue":3.4028234663852886e+38,"GRIB_name":"Mean wave direction","GRIB_numberOfPoints":405900,"GRIB_paramId":140230,"GRIB_shortName":"mwd","GRIB_stepType":"instant","GRIB_stepUnits":1,"GRIB_typeOfLevel":"meanSea","GRIB_units":"Degree true","_ARRAY_DIMENSIONS":["time","latitude","longitude"],"long_name":"Mean wave direction","standard_name":"unknown","units":"Degree true"}',
+  'mwp/.zarray': '{"chunks":[1,451,900],"compressor":null,"dtype":"<f8","fill_value":null,"filters":[{"dtype":"float64","id":"grib","var":"mwp"}],"order":"C","shape":[1,451,900],"zarr_format":2}',
+  'mwp/.zattrs': '{"GRIB_NV":0,"GRIB_Nx":900,"GRIB_Ny":451,"GRIB_cfName":"unknown","GRIB_cfVarName":"mwp","GRIB_dataType":"fc","GRIB_gridDefinitionDescription":"Latitude\\/longitude","GRIB_gridType":"regular_ll","GRIB_iDirectionIncrementInDegrees":0.4,"GRIB_iScansNegatively":0,"GRIB_jDirectionIncrementInDegrees":0.4,"GRIB_jPointsAreConsecutive":0,"GRIB_jScansPositively":0,"GRIB_latitudeOfFirstGridPointInDegrees":90.0,"GRIB_latitudeOfLastGridPointInDegrees":-90.0,"GRIB_longitudeOfFirstGridPointInDegrees":180.0,"GRIB_longitudeOfLastGridPointInDegrees":179.6,"GRIB_missingValue":3.4028234663852886e+38,"GRIB_name":"Mean wave period","GRIB_numberOfPoints":405900,"GRIB_paramId":140232,"GRIB_shortName":"mwp","GRIB_stepType":"instant","GRIB_stepUnits":1,"GRIB_typeOfLevel":"meanSea","GRIB_units":"s","_ARRAY_DIMENSIONS":["time","latitude","longitude"],"long_name":"Mean wave period","standard_name":"unknown","units":"s"}',
+  'swh/.zarray': '{"chunks":[1,451,900],"compressor":null,"dtype":"<f8","fill_value":null,"filters":[{"dtype":"float64","id":"grib","var":"swh"}],"order":"C","shape":[1,451,900],"zarr_format":2}',
+  'swh/.zattrs': '{"GRIB_NV":0,"GRIB_Nx":900,"GRIB_Ny":451,"GRIB_cfName":"unknown","GRIB_cfVarName":"swh","GRIB_dataType":"fc","GRIB_gridDefinitionDescription":"Latitude\\/longitude","GRIB_gridType":"regular_ll","GRIB_iDirectionIncrementInDegrees":0.4,"GRIB_iScansNegatively":0,"GRIB_jDirectionIncrementInDegrees":0.4,"GRIB_jPointsAreConsecutive":0,"GRIB_jScansPositively":0,"GRIB_latitudeOfFirstGridPointInDegrees":90.0,"GRIB_latitudeOfLastGridPointInDegrees":-90.0,"GRIB_longitudeOfFirstGridPointInDegrees":180.0,"GRIB_longitudeOfLastGridPointInDegrees":179.6,"GRIB_missingValue":3.4028234663852886e+38,"GRIB_name":"Significant height of combined wind waves and swell","GRIB_numberOfPoints":405900,"GRIB_paramId":140229,"GRIB_shortName":"swh","GRIB_stepType":"instant","GRIB_stepUnits":1,"GRIB_typeOfLevel":"meanSea","GRIB_units":"m","_ARRAY_DIMENSIONS":["time","latitude","longitude"],"long_name":"Significant height of combined wind waves and swell","standard_name":"unknown","units":"m"}',
+  'pp1d/.zarray': '{"chunks":[1,451,900],"compressor":null,"dtype":"<f8","fill_value":null,"filters":[{"dtype":"float64","id":"grib","var":"pp1d"}],"order":"C","shape":[1,451,900],"zarr_format":2}',
+  'pp1d/.zattrs': '{"GRIB_NV":0,"GRIB_Nx":900,"GRIB_Ny":451,"GRIB_cfName":"unknown","GRIB_cfVarName":"pp1d","GRIB_dataType":"fc","GRIB_gridDefinitionDescription":"Latitude\\/longitude","GRIB_gridType":"regular_ll","GRIB_iDirectionIncrementInDegrees":0.4,"GRIB_iScansNegatively":0,"GRIB_jDirectionIncrementInDegrees":0.4,"GRIB_jPointsAreConsecutive":0,"GRIB_jScansPositively":0,"GRIB_latitudeOfFirstGridPointInDegrees":90.0,"GRIB_latitudeOfLastGridPointInDegrees":-90.0,"GRIB_longitudeOfFirstGridPointInDegrees":180.0,"GRIB_longitudeOfLastGridPointInDegrees":179.6,"GRIB_missingValue":3.4028234663852886e+38,"GRIB_name":"Peak wave period","GRIB_numberOfPoints":405900,"GRIB_paramId":140231,"GRIB_shortName":"pp1d","GRIB_stepType":"instant","GRIB_stepUnits":1,"GRIB_typeOfLevel":"meanSea","GRIB_units":"s","_ARRAY_DIMENSIONS":["time","latitude","longitude"],"long_name":"Peak wave period","standard_name":"unknown","units":"s"}',
+  'step/.zarray': '{"chunks":[1],"compressor":null,"dtype":"<f8","fill_value":null,"filters":null,"order":"C","shape":[1],"zarr_format":2}',
+  'step/.zattrs': '{"_ARRAY_DIMENSIONS":["time"],"long_name":"time since forecast_reference_time","standard_name":"forecast_period","units":"hours"}',
+  'time/.zarray': '{\n    "chunks": [\n        1\n    ],\n    "compressor": null,\n    "dtype": "<i8",\n    "fill_value": null,\n    "filters": null,\n    "order": "C",\n    "shape": [\n        1\n    ],\n    "zarr_format": 2\n}',
+  'time/.zattrs': '{\n    "_ARRAY_DIMENSIONS": [\n        "time"\n    ],\n    "calendar": "proleptic_gregorian",\n    "long_name": "initial time of forecast",\n    "standard_name": "forecast_reference_time",\n    "units": "seconds since 1970-01-01T00:00:00"\n}',
+  'meanSea/.zarray': '{"chunks":[1],"compressor":null,"dtype":"<f8","fill_value":null,"filters":null,"order":"C","shape":[1],"zarr_format":2}',
+  'meanSea/.zattrs': '{"_ARRAY_DIMENSIONS":["time"]}',
+  'latitude/.zarray': '{"chunks":[451],"compressor":null,"dtype":"<f8","fill_value":null,"filters":[{"id": "range"}],"order":"C","shape":[451],"zarr_format":2}',
+  'latitude/.zattrs': '{"_ARRAY_DIMENSIONS":["latitude"],"long_name":"latitude","standard_name":"latitude","units":"degrees_north"}',
+  'longitude/.zarray': '{"chunks":[900],"compressor":null,"dtype":"<f8","fill_value":null,"filters":[{"id": "range"}],"order":"C","shape":[900],"zarr_format":2}',
+  'longitude/.zattrs': '{"_ARRAY_DIMENSIONS":["longitude"],"long_name":"longitude","standard_name":"longitude","units":"degrees_east"}',
+  'valid_time/.zarray': '{"chunks":[1],"compressor":null,"dtype":"<i8","fill_value":null,"filters":null,"order":"C","shape":[1],"zarr_format":2}',
+  'valid_time/.zattrs': '{"_ARRAY_DIMENSIONS":["time"],"calendar":"proleptic_gregorian","long_name":"time","standard_name":"time","units":"seconds since 1970-01-01T00:00:00"}'},
+ 'version': 1}, 
                 "title": "GRIB2 data file",
                 "description": (
                     "The forecast data, as a grib2 file. Subsets of the data can be loaded "