From 33c1018a2274278857ffd41daea5ab97842c8d60 Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 11 Feb 2025 17:10:51 +0100 Subject: [PATCH 1/7] add wave support for param 140251 --- polytope_feature/datacube/backends/fdb.py | 23 ++++- tests/test_wave_spectra_data.py | 101 ++++++++++++++++++++++ 2 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 tests/test_wave_spectra_data.py diff --git a/polytope_feature/datacube/backends/fdb.py b/polytope_feature/datacube/backends/fdb.py index bd60665a..803096c6 100644 --- a/polytope_feature/datacube/backends/fdb.py +++ b/polytope_feature/datacube/backends/fdb.py @@ -31,6 +31,7 @@ def __init__( if len(alternative_axes) == 0: logging.info("Find GribJump axes for %s", context) self.fdb_coordinates = self.gj.axes(partial_request, ctx=context) + print(self.fdb_coordinates) logging.info("Retrieved available GribJump axes for %s", context) if len(self.fdb_coordinates) == 0: raise BadRequestError(partial_request) @@ -80,10 +81,25 @@ def check_branching_axes(self, request): (upper, lower, idx) = polytope.extents(ax) if "sfc" in polytope.points[idx]: self.fdb_coordinates.pop("levelist", None) + # if ax == "stream": + # (upper, lower, idx) = polytope.extents(ax) + # if "wave" in polytope.points[idx]: + # if len(polytope.points[idx]) > 1: + # raise ValueError( + # "Please request stream wave separately from data on other streams.") # noqa: E501 + # self.fdb_coordinates.pop("levtype", None) + + if ax == "param": + (upper, lower, idx) = polytope.extents(ax) + if "140251" not in polytope.points[idx]: + self.fdb_coordinates.pop("direction", None) + self.fdb_coordinates.pop("frequency", None) + else: + # special param with direction and frequency + if len(polytope.points[idx]) > 1: + raise ValueError( + "Param 251 is part of a special branching of the datacube. Please request it separately.") # noqa: E501 self.fdb_coordinates.pop("quantile", None) - # TODO: When do these not appear?? - self.fdb_coordinates.pop("direction", None) - self.fdb_coordinates.pop("frequency", None) # NOTE: verify that we also remove the axis object for axes we've removed here axes_to_remove = set(self.complete_axes) - set(self.fdb_coordinates.keys()) @@ -129,6 +145,7 @@ def get(self, requests: TensorIndexTree, context=None): logging.debug("The requests we give GribJump are: %s", printed_list_to_gj) logging.info("Requests given to GribJump extract for %s", context) try: + print(complete_list_complete_uncompressed_requests) output_values = self.gj.extract(complete_list_complete_uncompressed_requests, context) except Exception as e: if "BadValue: Grid hash mismatch" in str(e): diff --git a/tests/test_wave_spectra_data.py b/tests/test_wave_spectra_data.py new file mode 100644 index 00000000..a9fcb10c --- /dev/null +++ b/tests/test_wave_spectra_data.py @@ -0,0 +1,101 @@ +import pytest +from helper_functions import find_nearest_latlon + +from polytope_feature.engine.hullslicer import HullSlicer +from polytope_feature.polytope import Polytope, Request +from polytope_feature.shapes import Box, Select +import pandas as pd + + +class TestHealpixGrid: + def setup_method(self, method): + + import pygribjump as gj + + self.fdb_datacube = gj.GribJump() + + self.options = { + "axis_config": [ + {"axis_name": "number", "transformations": [{"name": "type_change", "type": "int"}]}, + {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]}, + { + "axis_name": "date", + "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}], + }, + { + "axis_name": "values", + "transformations": [ + {"name": "mapper", "type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + ], + }, + {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]}, + {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, + ], + "pre_path": {"class": "od", "expver": "0001", "type": "fc", "stream": "wave", "date": "20250201"}, + "compressed_axes_config": [ + "longitude", + "latitude", + "levtype", + "step", + "date", + "domain", + "expver", + "param", + "class", + "stream", + "type", + "number", + ], + } + self.slicer = HullSlicer() + self.API = Polytope( + datacube=self.fdb_datacube, + engine=self.slicer, + options=self.options, + ) + + @pytest.mark.internet + def test_healpix_grid(self): + + request = Request( + Select("step", [1]), + Select("date", [pd.Timestamp("20250201T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["140251"]), + Select("class", ["od"]), + Select("stream", ["wave"]), + Select("type", ["fc"]), + Select("direction", ["1"]), + Select("frequency", ["1"]), + Box(["latitude", "longitude"], [1, 1], [2, 2]), + Select("levtype", ["sfc"]) + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 14 + assert result.leaves[0].result[1].size == 1 + assert result.leaves[1].result[1].size == 1 + + lats = [] + lons = [] + eccodes_lats = [] + tol = 1e-8 + leaves = result.leaves + for i, leaf in enumerate(leaves): + cubepath = leaf.flatten() + lat = cubepath["latitude"][0] + new_lons = cubepath["longitude"] + for j, lon in enumerate(new_lons): + lats.append(lat) + lons.append(lon) + nearest_points = find_nearest_latlon("./tests/data/wave_spectra.grib", lat, lon) + eccodes_lat = nearest_points[0][0]["lat"] + eccodes_lon = nearest_points[0][0]["lon"] + assert eccodes_lat - tol <= lat + assert lat <= eccodes_lat + tol + assert eccodes_lon - tol <= lon + assert lon <= eccodes_lon + tol + tol = 1e-2 + eccodes_lats.append(lat) + assert len(eccodes_lats) == 14 From 8ca745a9ae6f20289549f0de9f246a9a7c934db2 Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 11 Feb 2025 17:11:57 +0100 Subject: [PATCH 2/7] clean up --- polytope_feature/datacube/backends/fdb.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/polytope_feature/datacube/backends/fdb.py b/polytope_feature/datacube/backends/fdb.py index 803096c6..2900a613 100644 --- a/polytope_feature/datacube/backends/fdb.py +++ b/polytope_feature/datacube/backends/fdb.py @@ -81,13 +81,6 @@ def check_branching_axes(self, request): (upper, lower, idx) = polytope.extents(ax) if "sfc" in polytope.points[idx]: self.fdb_coordinates.pop("levelist", None) - # if ax == "stream": - # (upper, lower, idx) = polytope.extents(ax) - # if "wave" in polytope.points[idx]: - # if len(polytope.points[idx]) > 1: - # raise ValueError( - # "Please request stream wave separately from data on other streams.") # noqa: E501 - # self.fdb_coordinates.pop("levtype", None) if ax == "param": (upper, lower, idx) = polytope.extents(ax) From fe6e3e1771df9a1ceabbfdfdd3a79052c9be0896 Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 11 Feb 2025 17:13:22 +0100 Subject: [PATCH 3/7] isort + black --- polytope_feature/datacube/backends/fdb.py | 3 ++- tests/test_wave_spectra_data.py | 6 ++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/polytope_feature/datacube/backends/fdb.py b/polytope_feature/datacube/backends/fdb.py index 2900a613..95f769c8 100644 --- a/polytope_feature/datacube/backends/fdb.py +++ b/polytope_feature/datacube/backends/fdb.py @@ -91,7 +91,8 @@ def check_branching_axes(self, request): # special param with direction and frequency if len(polytope.points[idx]) > 1: raise ValueError( - "Param 251 is part of a special branching of the datacube. Please request it separately.") # noqa: E501 + "Param 251 is part of a special branching of the datacube. Please request it separately." + ) # noqa: E501 self.fdb_coordinates.pop("quantile", None) # NOTE: verify that we also remove the axis object for axes we've removed here diff --git a/tests/test_wave_spectra_data.py b/tests/test_wave_spectra_data.py index a9fcb10c..9e868b5d 100644 --- a/tests/test_wave_spectra_data.py +++ b/tests/test_wave_spectra_data.py @@ -1,15 +1,14 @@ +import pandas as pd import pytest from helper_functions import find_nearest_latlon from polytope_feature.engine.hullslicer import HullSlicer from polytope_feature.polytope import Polytope, Request from polytope_feature.shapes import Box, Select -import pandas as pd class TestHealpixGrid: def setup_method(self, method): - import pygribjump as gj self.fdb_datacube = gj.GribJump() @@ -56,7 +55,6 @@ def setup_method(self, method): @pytest.mark.internet def test_healpix_grid(self): - request = Request( Select("step", [1]), Select("date", [pd.Timestamp("20250201T000000")]), @@ -69,7 +67,7 @@ def test_healpix_grid(self): Select("direction", ["1"]), Select("frequency", ["1"]), Box(["latitude", "longitude"], [1, 1], [2, 2]), - Select("levtype", ["sfc"]) + Select("levtype", ["sfc"]), ) result = self.API.retrieve(request) result.pprint() From e24ea285f1816a14e18823de28776bfdb98c3b93 Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 11 Feb 2025 17:15:28 +0100 Subject: [PATCH 4/7] fix flake8 --- polytope_feature/datacube/backends/fdb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polytope_feature/datacube/backends/fdb.py b/polytope_feature/datacube/backends/fdb.py index 95f769c8..ab7a2dd1 100644 --- a/polytope_feature/datacube/backends/fdb.py +++ b/polytope_feature/datacube/backends/fdb.py @@ -91,8 +91,8 @@ def check_branching_axes(self, request): # special param with direction and frequency if len(polytope.points[idx]) > 1: raise ValueError( - "Param 251 is part of a special branching of the datacube. Please request it separately." - ) # noqa: E501 + "Param 251 is part of a special branching of the datacube. Please request it separately." # noqa: E501 + ) self.fdb_coordinates.pop("quantile", None) # NOTE: verify that we also remove the axis object for axes we've removed here From 062df44680e9535595ad5a23d787a815d85bb151 Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 11 Feb 2025 17:24:06 +0100 Subject: [PATCH 5/7] fix module not found for pygribjump in ci --- tests/test_wave_spectra_data.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_wave_spectra_data.py b/tests/test_wave_spectra_data.py index 9e868b5d..5eb1e32d 100644 --- a/tests/test_wave_spectra_data.py +++ b/tests/test_wave_spectra_data.py @@ -9,10 +9,6 @@ class TestHealpixGrid: def setup_method(self, method): - import pygribjump as gj - - self.fdb_datacube = gj.GribJump() - self.options = { "axis_config": [ {"axis_name": "number", "transformations": [{"name": "type_change", "type": "int"}]}, @@ -47,14 +43,18 @@ def setup_method(self, method): ], } self.slicer = HullSlicer() + + @pytest.mark.internet + def test_healpix_grid(self): + import pygribjump as gj + + self.fdb_datacube = gj.GribJump() self.API = Polytope( datacube=self.fdb_datacube, engine=self.slicer, options=self.options, ) - @pytest.mark.internet - def test_healpix_grid(self): request = Request( Select("step", [1]), Select("date", [pd.Timestamp("20250201T000000")]), From a00c861c51ccb04fa10aba7f89d7a6b9eb3f13df Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 12 Feb 2025 09:21:14 +0100 Subject: [PATCH 6/7] mark the test as fdb test --- tests/test_wave_spectra_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_wave_spectra_data.py b/tests/test_wave_spectra_data.py index 5eb1e32d..6835fee4 100644 --- a/tests/test_wave_spectra_data.py +++ b/tests/test_wave_spectra_data.py @@ -7,7 +7,7 @@ from polytope_feature.shapes import Box, Select -class TestHealpixGrid: +class TestWaveData: def setup_method(self, method): self.options = { "axis_config": [ @@ -44,7 +44,7 @@ def setup_method(self, method): } self.slicer = HullSlicer() - @pytest.mark.internet + @pytest.mark.fdb def test_healpix_grid(self): import pygribjump as gj From 6df162f1fbe71a50242fb322a8754fab8a907514 Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 12 Feb 2025 16:19:47 +0100 Subject: [PATCH 7/7] remove print statements --- polytope_feature/datacube/backends/fdb.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/polytope_feature/datacube/backends/fdb.py b/polytope_feature/datacube/backends/fdb.py index ab7a2dd1..609b74cf 100644 --- a/polytope_feature/datacube/backends/fdb.py +++ b/polytope_feature/datacube/backends/fdb.py @@ -31,7 +31,6 @@ def __init__( if len(alternative_axes) == 0: logging.info("Find GribJump axes for %s", context) self.fdb_coordinates = self.gj.axes(partial_request, ctx=context) - print(self.fdb_coordinates) logging.info("Retrieved available GribJump axes for %s", context) if len(self.fdb_coordinates) == 0: raise BadRequestError(partial_request) @@ -139,7 +138,6 @@ def get(self, requests: TensorIndexTree, context=None): logging.debug("The requests we give GribJump are: %s", printed_list_to_gj) logging.info("Requests given to GribJump extract for %s", context) try: - print(complete_list_complete_uncompressed_requests) output_values = self.gj.extract(complete_list_complete_uncompressed_requests, context) except Exception as e: if "BadValue: Grid hash mismatch" in str(e):