From bb7ebd827f9c73a198522e310207cd5f8ba3a838 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Wed, 28 Aug 2024 12:05:25 -0600 Subject: [PATCH 01/22] DAS-1934: Most basic implementation to allow consistent grids --- swath_projector/exceptions.py | 9 +++++++++ swath_projector/reproject.py | 9 +++++---- tests/unit/test_reproject.py | 8 +++++--- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/swath_projector/exceptions.py b/swath_projector/exceptions.py index 49f7d84..b908569 100644 --- a/swath_projector/exceptions.py +++ b/swath_projector/exceptions.py @@ -43,3 +43,12 @@ def __init__(self, missing_coordinate): 'MissingCoordinatesError', f'Could not find coordinate {missing_coordinate}.', ) + + +class InvalidTargetGrid(CustomError): + """Raised when a request specifies an incomplete or invalid grid.""" + + def __init__(self): + super().__init__( + 'InvalidTargetGrid', 'Insufficient or invalid target grid parameters.' + ) diff --git a/swath_projector/reproject.py b/swath_projector/reproject.py index dab1192..1bae12e 100644 --- a/swath_projector/reproject.py +++ b/swath_projector/reproject.py @@ -7,10 +7,12 @@ from typing import Dict from harmony.message import Message +from harmony.message_utility import has_self_consistent_grid from pyproj import Proj from varinfo import VarInfoFromNetCDF4 from swath_projector import nc_merge +from swath_projector.exceptions import InvalidTargetGrid from swath_projector.interpolation import resample_all_variables RADIUS_EARTH_METRES = ( @@ -123,13 +125,12 @@ def get_parameters_from_message( parameters['interpolation'] = INTERPOLATION_DEFAULT # ERROR 5: -tr and -ts options cannot be used at the same time. + if (parameters['xres'] is not None or parameters['yres'] is not None) and ( parameters['height'] is not None or parameters['width'] is not None ): - raise Exception( - '"scaleSize", "width" or/and "height" cannot ' - 'be used at the same time in the message.' - ) + if not has_self_consistent_grid(message): + raise InvalidTargetGrid() if not os.path.isfile(parameters['input_file']): raise Exception('Input file does not exist') diff --git a/tests/unit/test_reproject.py b/tests/unit/test_reproject.py index e3b566f..70a60d7 100644 --- a/tests/unit/test_reproject.py +++ b/tests/unit/test_reproject.py @@ -87,7 +87,7 @@ def test_get_parameters_error_5(self): dimensions, an exception is raised. """ - exception_snippet = 'cannot be used at the same time in the message.' + exception_snippet = 'Insufficient or invalid target grid parameters.' test_args = [ ['height and scaleSize', True, False, True, True], @@ -121,8 +121,10 @@ def test_get_parameters_error_5(self): with self.assertRaises(Exception) as context: get_parameters_from_message(message, self.granule_url, self.granule) - - self.assertTrue(str(context.exception).endswith(exception_snippet)) + self.assertTrue( + str(context.exception).endswith(exception_snippet), + f'Test Failed: {description}', + ) def test_get_parameters_missing_extents_or_dimensions(self): """Ensure that an exception is raised if there is only one of either From d072bcae242d018af5903ee72b3967905fab565f Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Thu, 29 Aug 2024 14:01:50 -0600 Subject: [PATCH 02/22] DAS-1934: Update comments and README for new grid validation. --- README.md | 12 ++++++++---- swath_projector/reproject.py | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d234842..bfaad5f 100644 --- a/README.md +++ b/README.md @@ -105,10 +105,14 @@ The Swath Projector can specify several options for reprojection in the CRS. If the `scaleExtent` is not specified, it is derived from the walking the perimeter of the input grid, reprojected those points to the target CRS, and finding the extreme values in each reprojected dimension. -* `scaleSize`: The resolution of each output pixel in the reprojected CRS. This - should not be specified if the `height` and `width` are also supplied. The - default values are derived from finding the total area of the swath via Gauss' - Area formula, and assuming the pixels are square. +* `scaleSize`: The resolution of each output pixel in the reprojected CRS. The + default values are derived from finding the total area of the swath via + Gauss' Area formula, and assuming the pixels are square. This should not + normally be specified if the `height` and `width` are also supplied, in this + case the grid definition must be internally consistent with itself. Where + consistency is determined by the equation `scaleSize = (scaleExtent.max - + scaleExtent.min) / dimension` + All the attributes in the `format` property are optional, and have defaults as described. diff --git a/swath_projector/reproject.py b/swath_projector/reproject.py index 1bae12e..808bf78 100644 --- a/swath_projector/reproject.py +++ b/swath_projector/reproject.py @@ -124,8 +124,8 @@ def get_parameters_from_message( if parameters['interpolation'] in [None, '', 'None']: parameters['interpolation'] = INTERPOLATION_DEFAULT - # ERROR 5: -tr and -ts options cannot be used at the same time. - + # when a user requests both a resolution and dimensions, then ensure the + # extents are consistent. if (parameters['xres'] is not None or parameters['yres'] is not None) and ( parameters['height'] is not None or parameters['width'] is not None ): From addf305774efa160416965c80c4cbd8308667fc0 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Thu, 29 Aug 2024 14:18:22 -0600 Subject: [PATCH 03/22] DAS-1934: Update changelog --- CHANGELOG.md | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c27abfe..79a508b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,22 @@ -## v1.0.1 -### 2024-04-05 +# Changelog + +## [v1.1.0] - 2024-08-29 +### Changed + +- [[DAS-1934](https://bugs.earthdata.nasa.gov/browse/DAS-1934)] + Input parameters that include both both resolutions (`xres` and `yres`) and + dimenions (`height` and `width`) no longer always raises an exception. An + exception is raised only when the parameters describe a grid that is not + internally consistent. [#14](https://github.com/nasa/harmony-swath-projector/pull/14) + + + +## v1.0.1 - 2024-04-05 This version of the Swath Projector implements black code formatting across the entire repository. There should be no functional changes to the service. -## v1.0.0 -### 2023-11-16 +## v1.0.0 - 2023-11-16 This version of the Harmony Swath Projector contains all functionality previously released internally to EOSDIS as `sds/swot-reproject:0.0.4`. @@ -23,3 +34,7 @@ Repository structure changes include: For more information on internal releases prior to NASA open-source approval, see legacy-CHANGELOG.md. + +[v1.1.0]:(https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.1) +[v1.0.1]:(https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.1) +[v1.0.0]:(https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.0) From 8150470e7cd4aea7b7cbb43fb64aef765fe3cfdc Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Thu, 29 Aug 2024 14:18:43 -0600 Subject: [PATCH 04/22] DAS-1934: Update changelog redux --- CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79a508b..5f73dd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,6 @@ exception is raised only when the parameters describe a grid that is not internally consistent. [#14](https://github.com/nasa/harmony-swath-projector/pull/14) - - ## v1.0.1 - 2024-04-05 This version of the Swath Projector implements black code formatting across the From 44589d8ce4174a83ec857a24105ef862c628829d Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Thu, 29 Aug 2024 14:48:21 -0600 Subject: [PATCH 05/22] DAS-1934: Update service version and address snyk vulnerabilities --- docker/service_version.txt | 2 +- docs/requirements.txt | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/service_version.txt b/docker/service_version.txt index 7dea76e..9084fa2 100644 --- a/docker/service_version.txt +++ b/docker/service_version.txt @@ -1 +1 @@ -1.0.1 +1.1.0 diff --git a/docs/requirements.txt b/docs/requirements.txt index efb479f..2ebeb36 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -10,7 +10,7 @@ # $ jupyter notebook. # 5) Click on the Swath Projector User Guide notebook. # -harmony-py ~= 0.4.9 -matplotlib ~= 3.7.2 -netCDF4 ~= 1.6.4 -notebook ~= 7.0.0 +harmony-py==0.4.15 +matplotlib==3.9.2 +netCDF4==1.7.1.post2 +notebook==7.2.2 From f9890f07f41c4a19bf451587419b2fa3506b6ad6 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Sun, 1 Sep 2024 09:42:56 -0600 Subject: [PATCH 06/22] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f73dd1..802619a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ - [[DAS-1934](https://bugs.earthdata.nasa.gov/browse/DAS-1934)] Input parameters that include both both resolutions (`xres` and `yres`) and - dimenions (`height` and `width`) no longer always raises an exception. An + dimenions (`height` and `width`) no longer always raise an exception. An exception is raised only when the parameters describe a grid that is not internally consistent. [#14](https://github.com/nasa/harmony-swath-projector/pull/14) From ed0ff0e7931b8c636826c8453ed8a27a8f8eb104 Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Sun, 1 Sep 2024 16:44:01 -0600 Subject: [PATCH 07/22] DAS-1934: Fix release note extraction script. Also updates CHANGELOG.md to add links for previous versions. --- CHANGELOG.md | 4 ++-- bin/extract-release-notes.sh | 25 ++++++++++++++++--------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 802619a..0586534 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,12 +9,12 @@ exception is raised only when the parameters describe a grid that is not internally consistent. [#14](https://github.com/nasa/harmony-swath-projector/pull/14) -## v1.0.1 - 2024-04-05 +## [v1.0.1] - 2024-04-05 This version of the Swath Projector implements black code formatting across the entire repository. There should be no functional changes to the service. -## v1.0.0 - 2023-11-16 +## [v1.0.0] - 2023-11-16 This version of the Harmony Swath Projector contains all functionality previously released internally to EOSDIS as `sds/swot-reproject:0.0.4`. diff --git a/bin/extract-release-notes.sh b/bin/extract-release-notes.sh index 00cbf40..a7cd3b5 100755 --- a/bin/extract-release-notes.sh +++ b/bin/extract-release-notes.sh @@ -7,17 +7,24 @@ # 2023-06-16: Created. # 2023-10-10: Copied from earthdata-varinfo repository to HOSS. # 2024-01-03: Copied from HOSS repository to the Swath Projector. +# 2024-09-01: Copied and modified from HyBIG repository to handle new format. # ############################################################################### CHANGELOG_FILE="CHANGELOG.md" -VERSION_PATTERN="^## v" -# Count number of versions in version file: -number_of_versions=$(grep -c "${VERSION_PATTERN}" ${CHANGELOG_FILE}) -if [ ${number_of_versions} -gt 1 ] -then - grep -B 9999 -m 2 "${VERSION_PATTERN}" ${CHANGELOG_FILE} | sed '$d' | sed '$d' -else - cat ${CHANGELOG_FILE} -fi +## captures versions +## >## v1.0.0 +## >## [v1.0.0] +VERSION_PATTERN="^## [\[]v" + +## captures url links +#[v1.0.0]:(https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.0) +LINK_PATTERN="^\[.*\].*/tag/.*" + +# Read the file and extract text between the first two occurrences of the +# VERSION_PATTERN +result=$(awk "/$VERSION_PATTERN/{c++; if(c==2) exit;} c==1" "$CHANGELOG_FILE") + +# Print the result +echo "$result" | grep -v "$VERSION_PATTERN" | grep -v "$LINK_PATTERN" From 7622aa7ac96536f4c4849d73b13410a9b0290f7a Mon Sep 17 00:00:00 2001 From: Matt Savoie Date: Mon, 9 Sep 2024 17:15:00 -0600 Subject: [PATCH 08/22] DAS-2216: Quick Fixes 1, 2 and 4 Doesn't fix new broken test --- bin/project_local_granule.py | 6 ++-- swath_projector/cf_config.json | 28 ++++++++++++++---- swath_projector/exceptions.py | 12 ++++++++ swath_projector/interpolation.py | 1 + swath_projector/nc_merge.py | 4 ++- swath_projector/utilities.py | 27 +++++++++++++++-- tests/unit/test_utilities.py | 51 ++++++++++++++++++++++++++++++++ 7 files changed, 116 insertions(+), 13 deletions(-) diff --git a/bin/project_local_granule.py b/bin/project_local_granule.py index 8924d08..4bb6445 100644 --- a/bin/project_local_granule.py +++ b/bin/project_local_granule.py @@ -125,8 +125,8 @@ def project_granule( { 'url': local_file_path, 'temporal': { - 'start': '2021-01-03T23:45:00.000Z', - 'end': '2020-01-04T00:00:00.000Z', + 'start': '2020-01-03T23:45:00.000Z', + 'end': '2025-01-04T00:00:00.000Z', }, 'bbox': [-180, -90, 180, 90], } @@ -141,5 +141,5 @@ def project_granule( reprojector = SwathProjectorAdapter(message, config=config(False)) - with patch('swotrepr.shutil.rmtree', side_effect=rmtree_side_effect): + with patch('swath_projector.adapter.shutil.rmtree', side_effect=rmtree_side_effect): reprojector.invoke() diff --git a/swath_projector/cf_config.json b/swath_projector/cf_config.json index c8575e9..f9b1d4a 100644 --- a/swath_projector/cf_config.json +++ b/swath_projector/cf_config.json @@ -1,11 +1,13 @@ { "Identification": "Swath Projector VarInfo configuration", - "Version": 2, + "Version": 3, "Collection_ShortName_Path": [ - "ShortName" + "ShortName", + "collection_shortname" ], "Mission": { - "VNP10": "VIIRS" + "VNP10": "VIIRS", + "TEMPO_O3TOT_L2": "TEMPO" }, "CF_Overrides": [ { @@ -17,10 +19,24 @@ "Attributes": [ { "Name": "coordinates", - "Value": "/GeolocationData/latitude, /GeolocationData/longitude" + "Value": "/GeolocationData/latitude /GeolocationData/longitude" } ], - "_Description": "VNP10 SnowData variables have incorrect relative paths for coordinates." + "_Description": "VNP10 SnowData variables have incorrect relative paths for coordinates." + }, + { + "Applicability": { + "Mission": "TEMPO", + "ShortNamePath": "TEMPO_03TOT_L2", + "Variable_Pattern": "/product/.*" + }, + "Attributes": [ + { + "Name": "coordinates", + "Value": "/Geolocation/latitude /Geolocation/longitude" + } + ], + "_Description": "TEMPO variables only have simple, single names for coordinates, but need full paths." } - ] + ] } diff --git a/swath_projector/exceptions.py b/swath_projector/exceptions.py index b908569..d8cf61d 100644 --- a/swath_projector/exceptions.py +++ b/swath_projector/exceptions.py @@ -52,3 +52,15 @@ def __init__(self): super().__init__( 'InvalidTargetGrid', 'Insufficient or invalid target grid parameters.' ) + + +class VariableShapeError(CustomError): + """Raised to skip resampling if variable shape does not match coordinates shape. + + While not optimal, raising this exception provides a temporary workaround + mismatched shapes for some science variables. + + """ + + def __init__(self): + super().__init__('VariableShapeError', 'Unable to resample variable.') diff --git a/swath_projector/interpolation.py b/swath_projector/interpolation.py index 4ab6d7b..cfa5a63 100644 --- a/swath_projector/interpolation.py +++ b/swath_projector/interpolation.py @@ -268,6 +268,7 @@ def get_ewa_results( ewa_information['target_area'], variable['values'], maximum_weight_mode=maximum_weight_mode, + rows_per_scan=2, # Added in QuickFix DAS-2216 to be fixed in DAS-2220 ) if variable['fill_value'] is not None: diff --git a/swath_projector/nc_merge.py b/swath_projector/nc_merge.py index b9fc615..9bb7b93 100644 --- a/swath_projector/nc_merge.py +++ b/swath_projector/nc_merge.py @@ -95,7 +95,9 @@ def create_output( else: logger.error(f'Cannot find "{dataset_file}".') - raise MissingReprojectedDataError(variable_name) + # QuickFix (DAS-2216) Ignore missing reprojections + logger.error(f'Not Including "{variable_name}" in "{output_file}".') + # raise MissingReprojectedDataError(variable_name) def set_output_attributes( diff --git a/swath_projector/utilities.py b/swath_projector/utilities.py index 35f8ed2..ad8ae75 100644 --- a/swath_projector/utilities.py +++ b/swath_projector/utilities.py @@ -43,10 +43,14 @@ def get_variable_values( return make_array_two_dimensional(variable[:]) elif 'time' in input_file.variables and 'time' in variable.dimensions: # Assumption: Array = (1, y, x) - return variable[0][:].filled(fill_value=fill_value) + return transpose_if_xdim_less_than_ydim( + variable[0][:].filled(fill_value=fill_value) + ) else: # Assumption: Array = (y, x) - return variable[:].filled(fill_value=fill_value) + return transpose_if_xdim_less_than_ydim( + variable[:].filled(fill_value=fill_value) + ) def get_coordinate_variable( @@ -62,7 +66,9 @@ def get_coordinate_variable( if coordinate_substring in coordinate.split('/')[-1] and variable_in_dataset( coordinate, dataset ): - return dataset[coordinate] + # QuickFix (DAS-2216) for short and wide swaths + return transpose_if_xdim_less_than_ydim(dataset[coordinate]) + raise MissingCoordinatesError(coordinates_tuple) @@ -216,3 +222,18 @@ def make_array_two_dimensional(one_dimensional_array: np.ndarray) -> np.ndarray: """ return np.expand_dims(one_dimensional_array, 1) + + +def transpose_if_xdim_less_than_ydim(variable: np.ndarray) -> np.ndarray: + """Return transposed variable when variable is wider than tall. + + QuickFix (DAS-2216): We presume that a swath has more rows than columns and + if that's not the case we transpose it so that it does. + """ + if variable.ndim != 2: + raise ValueError( + f'Input variable must be 2 dimensional, but got {variable.ndim} dimensions.' + ) + if variable.shape[0] < variable.shape[1]: + return np.ma.transpose(variable).copy() + return variable diff --git a/tests/unit/test_utilities.py b/tests/unit/test_utilities.py index abfcb1b..02d6709 100644 --- a/tests/unit/test_utilities.py +++ b/tests/unit/test_utilities.py @@ -16,6 +16,7 @@ get_variable_values, make_array_two_dimensional, qualify_reference, + transpose_if_xdim_less_than_ydim, variable_in_dataset, ) @@ -358,3 +359,53 @@ def test_make_array_two_dimensional(self): self.assertEqual(len(output_array.shape), 2) np.testing.assert_array_equal(output_array, expected_output) + + +class TestTransposeIfXdimLessThanYdim(TestCase): + + def test_wider_than_tall(self): + # Test case where x dim < y dim and should transpose + input_array = np.ma.array([[1, 2, 3], [4, 5, 6]]) + expected_output = np.ma.array([[1, 4], [2, 5], [3, 6]]) + result = transpose_if_xdim_less_than_ydim(input_array) + np.testing.assert_array_equal(result, expected_output) + self.assertEqual(result.shape, (3, 2)) + + def test_taller_than_wide(self): + # Test case where x < y and should not transpose. + input_array = np.ma.array([[1, 2], [3, 4], [5, 6]]) + result = transpose_if_xdim_less_than_ydim(input_array) + np.testing.assert_array_equal(result, input_array) + self.assertEqual(result.shape, (3, 2)) + + def test_square_array(self): + # test where y dim == x dim and should not transpose. + input_array = np.ma.array([[1, 2], [3, 4]]) + result = transpose_if_xdim_less_than_ydim(input_array) + np.testing.assert_array_equal(result, input_array) + self.assertEqual(result.shape, (2, 2)) + + def test_1d_array(self): + # Test case with a 1D array + input_array = np.ma.array([1, 2, 3]) + with self.assertRaisesRegex(ValueError, 'variable must be 2 dimensional'): + transpose_if_xdim_less_than_ydim(input_array) + + def test_3d_array(self): + # Test case with a 3D array + input_array = np.ma.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) + with self.assertRaisesRegex(ValueError, 'variable must be 2 dimensional'): + transpose_if_xdim_less_than_ydim(input_array) + + def test_masked_array(self): + # Test case with a masked array + input_array = np.ma.array( + [[1, 2, 3], [4, 5, 6]], mask=[[True, False, False], [False, True, False]] + ) + expected_output = np.ma.array( + [[1, 4], [2, 5], [3, 6]], + mask=[[True, False], [False, True], [False, False]], + ) + result = transpose_if_xdim_less_than_ydim(input_array) + np.testing.assert_array_equal(result, expected_output) + np.testing.assert_array_equal(result.mask, expected_output.mask) From 2a5a6efc7edf0a038c72f923311d7d7817202c35 Mon Sep 17 00:00:00 2001 From: Joey Schultz Date: Thu, 10 Oct 2024 10:03:39 -0400 Subject: [PATCH 09/22] DAS-2216: Modify earthdata-varinfo config for quick fix 1 --- swath_projector/earthdata_varinfo_config.json | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/swath_projector/earthdata_varinfo_config.json b/swath_projector/earthdata_varinfo_config.json index 66e5e23..8f5f27f 100644 --- a/swath_projector/earthdata_varinfo_config.json +++ b/swath_projector/earthdata_varinfo_config.json @@ -1,11 +1,10 @@ { "Identification": "Swath Projector VarInfo configuration", - "Version": 3, - "CollectionShortNamePath": [ - "ShortName" - ], + "Version": 4, + "CollectionShortNamePath": ["ShortName", "collection_shortname"], "Mission": { - "VNP10": "VIIRS" + "VNP10": "VIIRS", + "TEMPO_O3TOT_L2": "TEMPO" }, "MetadataOverrides": [ { @@ -21,6 +20,20 @@ } ], "_Description": "VNP10 SnowData variables have incorrect relative paths for coordinates." + }, + { + "Applicability": { + "Mission": "TEMPO", + "ShortNamePath": "TEMPO_O3TOT_L2", + "VariablePattern": "/product/.*|/support_data/.*" + }, + "Attributes": [ + { + "Name": "coordinates", + "Value": "/geolocation/latitude, /geolocation/longitude" + } + ], + "_Description": "TEMPO variables only have simple, single names for coordinates, but need full paths." } ] } From 447747566a53692d5d46812cb7d7d4c3e086a01f Mon Sep 17 00:00:00 2001 From: Joey Schultz Date: Thu, 10 Oct 2024 11:24:38 -0400 Subject: [PATCH 10/22] DAS-2216: Resolve unit tests that were failing due to quick fixes --- swath_projector/utilities.py | 3 +++ tests/unit/test_interpolation.py | 5 +++++ tests/unit/test_nc_merge.py | 3 ++- tests/unit/test_utilities.py | 2 +- 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/swath_projector/utilities.py b/swath_projector/utilities.py index ad8ae75..e57f92e 100644 --- a/swath_projector/utilities.py +++ b/swath_projector/utilities.py @@ -67,6 +67,9 @@ def get_coordinate_variable( coordinate, dataset ): # QuickFix (DAS-2216) for short and wide swaths + if dataset[coordinate].ndim == 1: + return dataset[coordinate] + return transpose_if_xdim_less_than_ydim(dataset[coordinate]) raise MissingCoordinatesError(coordinates_tuple) diff --git a/tests/unit/test_interpolation.py b/tests/unit/test_interpolation.py index 1203af2..501458d 100644 --- a/tests/unit/test_interpolation.py +++ b/tests/unit/test_interpolation.py @@ -384,6 +384,7 @@ def test_resample_ewa( self.mock_target_area, mock_values, maximum_weight_mode=False, + rows_per_scan=2 # Added in QuickFix DAS-2216 to be fixed in DAS-2220 ) mock_write_output.assert_called_once_with( self.mock_target_area, @@ -423,6 +424,7 @@ def test_resample_ewa( self.mock_target_area, mock_values, maximum_weight_mode=False, + rows_per_scan=2 # Added in QuickFix DAS-2216 to be fixed in DAS-2220 ) mock_write_output.assert_called_once_with( self.mock_target_area, @@ -491,6 +493,7 @@ def test_resample_ewa_nn( self.mock_target_area, mock_values, maximum_weight_mode=True, + rows_per_scan=2 # Added in QuickFix DAS-2216 to be fixed in DAS-2220 ) mock_write_output.assert_called_once_with( self.mock_target_area, @@ -530,6 +533,7 @@ def test_resample_ewa_nn( self.mock_target_area, mock_values, maximum_weight_mode=True, + rows_per_scan=2 # Added in QuickFix DAS-2216 to be fixed in DAS-2220 ) mock_write_output.assert_called_once_with( self.mock_target_area, @@ -581,6 +585,7 @@ def test_resample_ewa_nn( harmony_target_area, mock_values, maximum_weight_mode=True, + rows_per_scan=2 # Added in QuickFix DAS-2216 to be fixed in DAS-2220 ) # The Harmony target area should be given to the output function diff --git a/tests/unit/test_nc_merge.py b/tests/unit/test_nc_merge.py index e5c5421..dace742 100644 --- a/tests/unit/test_nc_merge.py +++ b/tests/unit/test_nc_merge.py @@ -2,7 +2,7 @@ import logging import os from datetime import datetime -from unittest import TestCase +from unittest import TestCase, skip from unittest.mock import Mock, patch from netCDF4 import Dataset @@ -186,6 +186,7 @@ def test_same_data_type(self): output_data_type = out_dataset[test_variable].datatype self.assertEqual(input_data_type, output_data_type, 'Should be equal') + @skip("Test disabled for DAS-2216 quick fix that ignores missing reprojections") def test_missing_file_raises_error(self): """If a science variable should be included in the output, but there is no associated output file, an exception should be raised. diff --git a/tests/unit/test_utilities.py b/tests/unit/test_utilities.py index 02d6709..879a7f7 100644 --- a/tests/unit/test_utilities.py +++ b/tests/unit/test_utilities.py @@ -80,7 +80,7 @@ def test_get_variable_values(self): wind_speed_values = get_variable_values(dataset, wind_speed, None) self.assertIsInstance(wind_speed_values, np.ndarray) self.assertEqual(len(wind_speed_values.shape), 2) - self.assertEqual(wind_speed_values.shape, wind_speed.shape) + self.assertEqual(wind_speed_values.shape, np.ma.transpose(wind_speed).shape) with self.subTest('Masked values are set to fill value.'): fill_value = 210 From 404088c0f5fbab602a9781cbe804389318520257 Mon Sep 17 00:00:00 2001 From: Joey Schultz Date: Thu, 10 Oct 2024 11:46:20 -0400 Subject: [PATCH 11/22] DAS-2216: Update service version and CHANGELOG. --- CHANGELOG.md | 23 ++++++++++++++++------- docker/service_version.txt | 2 +- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3504dd7..e5b7200 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ # Changelog +## [v1.2.0] - 2024-10-10 + +### Changed + +- [[DAS-2216](https://bugs.earthdata.nasa.gov/browse/DAS-2216)] + The Swath Projector has been updated with quick fixes to add support for TEMPO level 2 data. + ## [v1.1.1] - 2024-09-16 + ### Changed - [[TRT-558](https://bugs.earthdata.nasa.gov/browse/TRT-558)] @@ -12,6 +20,7 @@ also been renamed to `earthdata_varinfo_config.json`. ## [v1.1.0] - 2024-08-29 + ### Changed - [[DAS-1934](https://bugs.earthdata.nasa.gov/browse/DAS-1934)] @@ -37,14 +46,14 @@ include updated documentation and files outlined by the Repository structure changes include: -* Migrating `pymods` directory to `swath_projector`. -* Migrating `swotrepr.py` to `swath_projector/adapter.py`. -* Addition of `swath_projector/main.py`. +- Migrating `pymods` directory to `swath_projector`. +- Migrating `swotrepr.py` to `swath_projector/adapter.py`. +- Addition of `swath_projector/main.py`. For more information on internal releases prior to NASA open-source approval, see legacy-CHANGELOG.md. -[v1.1.1]:(https://github.com/nasa/harmony-swath-projector/releases/tag/1.1.0) -[v1.1.0]:(https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.1) -[v1.0.1]:(https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.1) -[v1.0.0]:(https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.0) +[v1.1.1]: (https://github.com/nasa/harmony-swath-projector/releases/tag/1.1.0) +[v1.1.0]: (https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.1) +[v1.0.1]: (https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.1) +[v1.0.0]: (https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.0) diff --git a/docker/service_version.txt b/docker/service_version.txt index 524cb55..26aaba0 100644 --- a/docker/service_version.txt +++ b/docker/service_version.txt @@ -1 +1 @@ -1.1.1 +1.2.0 From 72223b99ceb60347f281a9eb833ce7626671a63c Mon Sep 17 00:00:00 2001 From: Joey Schultz Date: Wed, 16 Oct 2024 15:19:37 -0400 Subject: [PATCH 12/22] Modify transpose_if_xdim_less_than_ydim to resolve mask array not being returned for coordinate variables --- swath_projector/utilities.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/swath_projector/utilities.py b/swath_projector/utilities.py index e57f92e..944f15e 100644 --- a/swath_projector/utilities.py +++ b/swath_projector/utilities.py @@ -238,5 +238,11 @@ def transpose_if_xdim_less_than_ydim(variable: np.ndarray) -> np.ndarray: f'Input variable must be 2 dimensional, but got {variable.ndim} dimensions.' ) if variable.shape[0] < variable.shape[1]: - return np.ma.transpose(variable).copy() + transposed_variable = np.ma.transpose(variable).copy() + if hasattr(variable, 'mask'): + transposed_variable.mask = np.ma.transpose(variable[:].mask) + return transposed_variable + + return transposed_variable + return variable From 656c85819a065b7ebe5645792d6bd189daf74bb9 Mon Sep 17 00:00:00 2001 From: Joey Schultz Date: Wed, 16 Oct 2024 15:22:44 -0400 Subject: [PATCH 13/22] Create notebook for PR testing and demo purposes --- docs/TEMPO_O3TOT_L2_example.ipynb | 218 ++++++++++++++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 docs/TEMPO_O3TOT_L2_example.ipynb diff --git a/docs/TEMPO_O3TOT_L2_example.ipynb b/docs/TEMPO_O3TOT_L2_example.ipynb new file mode 100644 index 0000000..523d534 --- /dev/null +++ b/docs/TEMPO_O3TOT_L2_example.ipynb @@ -0,0 +1,218 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# TEMPO_O3TOT_L2 Harmony Tests" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Environment setup:\n", + "\n", + "This notebook assumes that it is being run in a local Python environment, configured using either `pyenv` or conda. Either can be used, but the dependencies will be installed via Pip. To install the required packages to run this notebook:\n", + "\n", + "```bash\n", + "$ pip install -r requirements.txt\n", + "```\n", + "\n", + "A `.netrc` file must also be located in the user's home directory." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Import Requirements:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from os.path import exists\n", + "from os import makedirs, replace\n", + "from os.path import exists, join as path_join\n", + "from shutil import rmtree\n", + "from harmony import Client, Environment, Collection, Request\n", + "from harmony.harmony import ProcessingFailedException" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Set the Harmony Environment:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "harmony_host_url = 'http://localhost:3000'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Identify Harmony environment (for easier reference):" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "host_environment = {'http://localhost:3000': Environment.LOCAL,\n", + " 'https://harmony.sit.earthdata.nasa.gov': Environment.SIT,\n", + " 'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n", + " 'https://harmony.earthdata.nasa.gov': Environment.PROD}\n", + "\n", + "\n", + "harmony_environment = host_environment.get(harmony_host_url)\n", + "\n", + "if harmony_environment is not None:\n", + " harmony_client = Client(env=harmony_environment)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### TEMPO_O3TOT_L2 Data:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "collection = Collection(id='C1270257471-EEDTEST')\n", + "granule_id='G1270257472-EEDTEST'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Building Swath Projector Request:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "harmony_client = Client(env=Environment.LOCAL)\n", + "request = Request(\n", + " crs='EPSG:4326',\n", + " interpolation='near',\n", + " collection=collection,\n", + " granule_id=granule_id\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Create output directory to save test results:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "collection_short_name = 'TEMPO_O3TOT_L2'\n", + "output_directory = path_join('output_files', collection_short_name) + '/'\n", + "\n", + "if exists(output_directory):\n", + " rmtree(output_directory)\n", + "\n", + "makedirs(output_directory)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Submitting Harmony Request:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def submit_and_download(harmony_client: Client, request: Request,\n", + " output_file_name: str):\n", + " \"\"\" Submit a Harmony request via a `harmony-py` client. Wait for the\n", + " Harmony job to finish, then download the results to the specified file\n", + " path.\n", + "\n", + " \"\"\"\n", + " downloaded_filename = None\n", + "\n", + " try:\n", + " job_id = harmony_client.submit(request)\n", + "\n", + " for filename in [file_future.result()\n", + " for file_future\n", + " in harmony_client.download_all(job_id,\n", + " overwrite=True)]:\n", + "\n", + " print(f'Downloaded: {filename}')\n", + " downloaded_filename = filename\n", + "\n", + " if downloaded_filename is not None:\n", + " replace(downloaded_filename, output_file_name)\n", + " print(f'Saved output to: {output_file_name}')\n", + "\n", + " except ProcessingFailedException:\n", + " print_error('Harmony request failed to complete successfully.')\n", + "\n", + "def print_error(error_string: str) -> str:\n", + " \"\"\"Print an error, with formatting for red text. \"\"\"\n", + " print(f'\\033[91m{error_string}\\033[0m')\n", + "\n", + "swath_output_file_name = output_directory + f'TEMPO_O3TOT_L2_{granule_id}_output.h5'\n", + "submit_and_download(harmony_client, request, swath_output_file_name)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "localharmony", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 3601f92c1745897ab7faf23e24da7b81c51eaf83 Mon Sep 17 00:00:00 2001 From: Joey Schultz Date: Wed, 16 Oct 2024 17:36:13 -0400 Subject: [PATCH 14/22] Install and run pre-commit --- docs/TEMPO_O3TOT_L2_example.ipynb | 47 ++++++++++++++++--------------- swath_projector/utilities.py | 2 +- tests/unit/test_interpolation.py | 10 +++---- tests/unit/test_utilities.py | 4 ++- 4 files changed, 34 insertions(+), 29 deletions(-) diff --git a/docs/TEMPO_O3TOT_L2_example.ipynb b/docs/TEMPO_O3TOT_L2_example.ipynb index 523d534..55384d3 100644 --- a/docs/TEMPO_O3TOT_L2_example.ipynb +++ b/docs/TEMPO_O3TOT_L2_example.ipynb @@ -35,11 +35,12 @@ "metadata": {}, "outputs": [], "source": [ - "from os.path import exists\n", "from os import makedirs, replace\n", - "from os.path import exists, join as path_join\n", + "from os.path import exists\n", + "from os.path import join as path_join\n", "from shutil import rmtree\n", - "from harmony import Client, Environment, Collection, Request\n", + "\n", + "from harmony import Client, Collection, Environment, Request\n", "from harmony.harmony import ProcessingFailedException" ] }, @@ -72,10 +73,12 @@ "metadata": {}, "outputs": [], "source": [ - "host_environment = {'http://localhost:3000': Environment.LOCAL,\n", - " 'https://harmony.sit.earthdata.nasa.gov': Environment.SIT,\n", - " 'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n", - " 'https://harmony.earthdata.nasa.gov': Environment.PROD}\n", + "host_environment = {\n", + " 'http://localhost:3000': Environment.LOCAL,\n", + " 'https://harmony.sit.earthdata.nasa.gov': Environment.SIT,\n", + " 'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n", + " 'https://harmony.earthdata.nasa.gov': Environment.PROD,\n", + "}\n", "\n", "\n", "harmony_environment = host_environment.get(harmony_host_url)\n", @@ -98,7 +101,7 @@ "outputs": [], "source": [ "collection = Collection(id='C1270257471-EEDTEST')\n", - "granule_id='G1270257472-EEDTEST'" + "granule_id = 'G1270257472-EEDTEST'" ] }, { @@ -116,10 +119,7 @@ "source": [ "harmony_client = Client(env=Environment.LOCAL)\n", "request = Request(\n", - " crs='EPSG:4326',\n", - " interpolation='near',\n", - " collection=collection,\n", - " granule_id=granule_id\n", + " crs='EPSG:4326', interpolation='near', collection=collection, granule_id=granule_id\n", ")" ] }, @@ -158,11 +158,12 @@ "metadata": {}, "outputs": [], "source": [ - "def submit_and_download(harmony_client: Client, request: Request,\n", - " output_file_name: str):\n", - " \"\"\" Submit a Harmony request via a `harmony-py` client. Wait for the\n", - " Harmony job to finish, then download the results to the specified file\n", - " path.\n", + "def submit_and_download(\n", + " harmony_client: Client, request: Request, output_file_name: str\n", + "):\n", + " \"\"\"Submit a Harmony request via a `harmony-py` client. Wait for the\n", + " Harmony job to finish, then download the results to the specified file\n", + " path.\n", "\n", " \"\"\"\n", " downloaded_filename = None\n", @@ -170,10 +171,10 @@ " try:\n", " job_id = harmony_client.submit(request)\n", "\n", - " for filename in [file_future.result()\n", - " for file_future\n", - " in harmony_client.download_all(job_id,\n", - " overwrite=True)]:\n", + " for filename in [\n", + " file_future.result()\n", + " for file_future in harmony_client.download_all(job_id, overwrite=True)\n", + " ]:\n", "\n", " print(f'Downloaded: {filename}')\n", " downloaded_filename = filename\n", @@ -185,10 +186,12 @@ " except ProcessingFailedException:\n", " print_error('Harmony request failed to complete successfully.')\n", "\n", + "\n", "def print_error(error_string: str) -> str:\n", - " \"\"\"Print an error, with formatting for red text. \"\"\"\n", + " \"\"\"Print an error, with formatting for red text.\"\"\"\n", " print(f'\\033[91m{error_string}\\033[0m')\n", "\n", + "\n", "swath_output_file_name = output_directory + f'TEMPO_O3TOT_L2_{granule_id}_output.h5'\n", "submit_and_download(harmony_client, request, swath_output_file_name)" ] diff --git a/swath_projector/utilities.py b/swath_projector/utilities.py index 944f15e..6d679c6 100644 --- a/swath_projector/utilities.py +++ b/swath_projector/utilities.py @@ -242,7 +242,7 @@ def transpose_if_xdim_less_than_ydim(variable: np.ndarray) -> np.ndarray: if hasattr(variable, 'mask'): transposed_variable.mask = np.ma.transpose(variable[:].mask) return transposed_variable - + return transposed_variable return variable diff --git a/tests/unit/test_interpolation.py b/tests/unit/test_interpolation.py index 501458d..5c7a094 100644 --- a/tests/unit/test_interpolation.py +++ b/tests/unit/test_interpolation.py @@ -384,7 +384,7 @@ def test_resample_ewa( self.mock_target_area, mock_values, maximum_weight_mode=False, - rows_per_scan=2 # Added in QuickFix DAS-2216 to be fixed in DAS-2220 + rows_per_scan=2, # Added in QuickFix DAS-2216 to be fixed in DAS-2220 ) mock_write_output.assert_called_once_with( self.mock_target_area, @@ -424,7 +424,7 @@ def test_resample_ewa( self.mock_target_area, mock_values, maximum_weight_mode=False, - rows_per_scan=2 # Added in QuickFix DAS-2216 to be fixed in DAS-2220 + rows_per_scan=2, # Added in QuickFix DAS-2216 to be fixed in DAS-2220 ) mock_write_output.assert_called_once_with( self.mock_target_area, @@ -493,7 +493,7 @@ def test_resample_ewa_nn( self.mock_target_area, mock_values, maximum_weight_mode=True, - rows_per_scan=2 # Added in QuickFix DAS-2216 to be fixed in DAS-2220 + rows_per_scan=2, # Added in QuickFix DAS-2216 to be fixed in DAS-2220 ) mock_write_output.assert_called_once_with( self.mock_target_area, @@ -533,7 +533,7 @@ def test_resample_ewa_nn( self.mock_target_area, mock_values, maximum_weight_mode=True, - rows_per_scan=2 # Added in QuickFix DAS-2216 to be fixed in DAS-2220 + rows_per_scan=2, # Added in QuickFix DAS-2216 to be fixed in DAS-2220 ) mock_write_output.assert_called_once_with( self.mock_target_area, @@ -585,7 +585,7 @@ def test_resample_ewa_nn( harmony_target_area, mock_values, maximum_weight_mode=True, - rows_per_scan=2 # Added in QuickFix DAS-2216 to be fixed in DAS-2220 + rows_per_scan=2, # Added in QuickFix DAS-2216 to be fixed in DAS-2220 ) # The Harmony target area should be given to the output function diff --git a/tests/unit/test_utilities.py b/tests/unit/test_utilities.py index 879a7f7..1d752b4 100644 --- a/tests/unit/test_utilities.py +++ b/tests/unit/test_utilities.py @@ -80,7 +80,9 @@ def test_get_variable_values(self): wind_speed_values = get_variable_values(dataset, wind_speed, None) self.assertIsInstance(wind_speed_values, np.ndarray) self.assertEqual(len(wind_speed_values.shape), 2) - self.assertEqual(wind_speed_values.shape, np.ma.transpose(wind_speed).shape) + self.assertEqual( + wind_speed_values.shape, np.ma.transpose(wind_speed).shape + ) with self.subTest('Masked values are set to fill value.'): fill_value = 210 From 735894d1f7c3771c9aa53047e64669af8db5f877 Mon Sep 17 00:00:00 2001 From: Joey Schultz Date: Mon, 21 Oct 2024 10:53:38 -0400 Subject: [PATCH 15/22] Remove unused exception, fix CHANGELOG links, and other minor updates --- CHANGELOG.md | 5 +++-- swath_projector/earthdata_varinfo_config.json | 5 ++++- swath_projector/exceptions.py | 12 ------------ swath_projector/nc_merge.py | 3 --- swath_projector/utilities.py | 1 - 5 files changed, 7 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5b7200..c3088ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,7 +53,8 @@ Repository structure changes include: For more information on internal releases prior to NASA open-source approval, see legacy-CHANGELOG.md. -[v1.1.1]: (https://github.com/nasa/harmony-swath-projector/releases/tag/1.1.0) -[v1.1.0]: (https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.1) +[v1.2.0]: (https://github.com/nasa/harmony-swath-projector/releases/tag/1.2.0) +[v1.1.1]: (https://github.com/nasa/harmony-swath-projector/releases/tag/1.1.1) +[v1.1.0]: (https://github.com/nasa/harmony-swath-projector/releases/tag/1.1.0) [v1.0.1]: (https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.1) [v1.0.0]: (https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.0) diff --git a/swath_projector/earthdata_varinfo_config.json b/swath_projector/earthdata_varinfo_config.json index 8f5f27f..a7b2dab 100644 --- a/swath_projector/earthdata_varinfo_config.json +++ b/swath_projector/earthdata_varinfo_config.json @@ -1,7 +1,10 @@ { "Identification": "Swath Projector VarInfo configuration", "Version": 4, - "CollectionShortNamePath": ["ShortName", "collection_shortname"], + "CollectionShortNamePath": [ + "ShortName", + "collection_shortname" + ], "Mission": { "VNP10": "VIIRS", "TEMPO_O3TOT_L2": "TEMPO" diff --git a/swath_projector/exceptions.py b/swath_projector/exceptions.py index d8cf61d..b908569 100644 --- a/swath_projector/exceptions.py +++ b/swath_projector/exceptions.py @@ -52,15 +52,3 @@ def __init__(self): super().__init__( 'InvalidTargetGrid', 'Insufficient or invalid target grid parameters.' ) - - -class VariableShapeError(CustomError): - """Raised to skip resampling if variable shape does not match coordinates shape. - - While not optimal, raising this exception provides a temporary workaround - mismatched shapes for some science variables. - - """ - - def __init__(self): - super().__init__('VariableShapeError', 'Unable to resample variable.') diff --git a/swath_projector/nc_merge.py b/swath_projector/nc_merge.py index 9bb7b93..dc8f5c6 100644 --- a/swath_projector/nc_merge.py +++ b/swath_projector/nc_merge.py @@ -13,7 +13,6 @@ from netCDF4 import Dataset, Variable from varinfo import VarInfoFromNetCDF4 -from swath_projector.exceptions import MissingReprojectedDataError from swath_projector.utilities import get_variable_file_path, variable_in_dataset # Values needed for history_json attribute @@ -96,8 +95,6 @@ def create_output( else: logger.error(f'Cannot find "{dataset_file}".') # QuickFix (DAS-2216) Ignore missing reprojections - logger.error(f'Not Including "{variable_name}" in "{output_file}".') - # raise MissingReprojectedDataError(variable_name) def set_output_attributes( diff --git a/swath_projector/utilities.py b/swath_projector/utilities.py index 6d679c6..3cec8cc 100644 --- a/swath_projector/utilities.py +++ b/swath_projector/utilities.py @@ -241,7 +241,6 @@ def transpose_if_xdim_less_than_ydim(variable: np.ndarray) -> np.ndarray: transposed_variable = np.ma.transpose(variable).copy() if hasattr(variable, 'mask'): transposed_variable.mask = np.ma.transpose(variable[:].mask) - return transposed_variable return transposed_variable From 8770da0e67d7b19218e2a98cdafebae532fcc554 Mon Sep 17 00:00:00 2001 From: Joey Schultz Date: Mon, 21 Oct 2024 13:40:58 -0400 Subject: [PATCH 16/22] Reorganize some of the notebook functions --- docs/TEMPO_O3TOT_L2_example.ipynb | 42 +++++++++++++++---------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/docs/TEMPO_O3TOT_L2_example.ipynb b/docs/TEMPO_O3TOT_L2_example.ipynb index 55384d3..d4ec1df 100644 --- a/docs/TEMPO_O3TOT_L2_example.ipynb +++ b/docs/TEMPO_O3TOT_L2_example.ipynb @@ -104,25 +104,6 @@ "granule_id = 'G1270257472-EEDTEST'" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Building Swath Projector Request:" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "harmony_client = Client(env=Environment.LOCAL)\n", - "request = Request(\n", - " crs='EPSG:4326', interpolation='near', collection=collection, granule_id=granule_id\n", - ")" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -149,7 +130,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "#### Submitting Harmony Request:" + "#### Utility functions for submitting Harmony Requests:" ] }, { @@ -189,8 +170,27 @@ "\n", "def print_error(error_string: str) -> str:\n", " \"\"\"Print an error, with formatting for red text.\"\"\"\n", - " print(f'\\033[91m{error_string}\\033[0m')\n", + " print(f'\\033[91m{error_string}\\033[0m')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Submitting Harmony Request:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "\n", + "harmony_client = Client(env=Environment.LOCAL)\n", + "request = Request(\n", + " crs='EPSG:4326', interpolation='near', collection=collection, granule_id=granule_id\n", + ")\n", "\n", "swath_output_file_name = output_directory + f'TEMPO_O3TOT_L2_{granule_id}_output.h5'\n", "submit_and_download(harmony_client, request, swath_output_file_name)" From 78dd9df6bb3ca5e408928a1c5cf3ac51a21004d7 Mon Sep 17 00:00:00 2001 From: Joey Schultz Date: Mon, 21 Oct 2024 14:44:45 -0400 Subject: [PATCH 17/22] Modify assumption comments in get_variable_values for clarity --- swath_projector/utilities.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swath_projector/utilities.py b/swath_projector/utilities.py index 3cec8cc..c513fdf 100644 --- a/swath_projector/utilities.py +++ b/swath_projector/utilities.py @@ -42,12 +42,12 @@ def get_variable_values( if len(variable[:].shape) == 1: return make_array_two_dimensional(variable[:]) elif 'time' in input_file.variables and 'time' in variable.dimensions: - # Assumption: Array = (1, y, x) + # Assumption: Array = (time, along-track, across-track) return transpose_if_xdim_less_than_ydim( variable[0][:].filled(fill_value=fill_value) ) else: - # Assumption: Array = (y, x) + # Assumption: Array = (along-track, across-track) return transpose_if_xdim_less_than_ydim( variable[:].filled(fill_value=fill_value) ) From 25cdc1098508cbccc841bb08293697d8cc0530b4 Mon Sep 17 00:00:00 2001 From: Joey Schultz Date: Wed, 23 Oct 2024 10:08:17 -0400 Subject: [PATCH 18/22] Add to varinfo config, re-enable MissingReprojectedDataError, modify docstrings --- swath_projector/earthdata_varinfo_config.json | 25 +++++++++++++++++-- swath_projector/nc_merge.py | 3 ++- swath_projector/utilities.py | 3 ++- tests/unit/test_nc_merge.py | 3 +-- tests/unit/test_utilities.py | 12 ++++----- 5 files changed, 34 insertions(+), 12 deletions(-) diff --git a/swath_projector/earthdata_varinfo_config.json b/swath_projector/earthdata_varinfo_config.json index a7b2dab..25069d4 100644 --- a/swath_projector/earthdata_varinfo_config.json +++ b/swath_projector/earthdata_varinfo_config.json @@ -9,6 +9,27 @@ "VNP10": "VIIRS", "TEMPO_O3TOT_L2": "TEMPO" }, + "ExcludedScienceVariables": [ + { + "Applicability": { + "Mission": "TEMPO", + "ShortNamePath": "TEMPO_O3TOT_L2" + }, + "VariablePattern": [ + "/support_data/a_priori_layer_o3", + "/support_data/cal_adjustment", + "/support_data/dNdR", + "/support_data/layer_efficiency", + "/support_data/lut_wavelength", + "/support_data/N_value", + "/support_data/N_value_residual", + "/support_data/ozone_sensitivity_ratio", + "/support_data/step_1_N_value_residual", + "/support_data/step_2_N_value_residual", + "/support_data/temp_sensitivity_ratio" + ] + } + ], "MetadataOverrides": [ { "Applicability": { @@ -28,7 +49,7 @@ "Applicability": { "Mission": "TEMPO", "ShortNamePath": "TEMPO_O3TOT_L2", - "VariablePattern": "/product/.*|/support_data/.*" + "VariablePattern": "^/product/.*|^/support_data/.*" }, "Attributes": [ { @@ -36,7 +57,7 @@ "Value": "/geolocation/latitude, /geolocation/longitude" } ], - "_Description": "TEMPO variables only have simple, single names for coordinates, but need full paths." + "_Description": "TEMPO_O3TOT_L2 variables only contain basenames for coordinates, which are found in sibling hierarchical groups. This rule fully qualifies the paths to these coordinates." } ] } diff --git a/swath_projector/nc_merge.py b/swath_projector/nc_merge.py index dc8f5c6..b9fc615 100644 --- a/swath_projector/nc_merge.py +++ b/swath_projector/nc_merge.py @@ -13,6 +13,7 @@ from netCDF4 import Dataset, Variable from varinfo import VarInfoFromNetCDF4 +from swath_projector.exceptions import MissingReprojectedDataError from swath_projector.utilities import get_variable_file_path, variable_in_dataset # Values needed for history_json attribute @@ -94,7 +95,7 @@ def create_output( else: logger.error(f'Cannot find "{dataset_file}".') - # QuickFix (DAS-2216) Ignore missing reprojections + raise MissingReprojectedDataError(variable_name) def set_output_attributes( diff --git a/swath_projector/utilities.py b/swath_projector/utilities.py index c513fdf..df67795 100644 --- a/swath_projector/utilities.py +++ b/swath_projector/utilities.py @@ -31,7 +31,8 @@ def get_variable_values( As the variable data are returned as a `numpy.ma.MaskedArray`, the will return no data in the filled pixels. To ensure that the data are correctly handled, the fill value is applied to masked pixels using the - `filled` method. + `filled` method. The variable values are transposed if the `along-track` + dimension size is less than the `across-track` dimension size. """ # TODO: Remove in favour of apply2D or process_subdimension. diff --git a/tests/unit/test_nc_merge.py b/tests/unit/test_nc_merge.py index dace742..e5c5421 100644 --- a/tests/unit/test_nc_merge.py +++ b/tests/unit/test_nc_merge.py @@ -2,7 +2,7 @@ import logging import os from datetime import datetime -from unittest import TestCase, skip +from unittest import TestCase from unittest.mock import Mock, patch from netCDF4 import Dataset @@ -186,7 +186,6 @@ def test_same_data_type(self): output_data_type = out_dataset[test_variable].datatype self.assertEqual(input_data_type, output_data_type, 'Should be equal') - @skip("Test disabled for DAS-2216 quick fix that ignores missing reprojections") def test_missing_file_raises_error(self): """If a science variable should be included in the output, but there is no associated output file, an exception should be raised. diff --git a/tests/unit/test_utilities.py b/tests/unit/test_utilities.py index 1d752b4..b31e00f 100644 --- a/tests/unit/test_utilities.py +++ b/tests/unit/test_utilities.py @@ -366,7 +366,7 @@ def test_make_array_two_dimensional(self): class TestTransposeIfXdimLessThanYdim(TestCase): def test_wider_than_tall(self): - # Test case where x dim < y dim and should transpose + """Test case where x dim < y dim and should transpose.""" input_array = np.ma.array([[1, 2, 3], [4, 5, 6]]) expected_output = np.ma.array([[1, 4], [2, 5], [3, 6]]) result = transpose_if_xdim_less_than_ydim(input_array) @@ -374,33 +374,33 @@ def test_wider_than_tall(self): self.assertEqual(result.shape, (3, 2)) def test_taller_than_wide(self): - # Test case where x < y and should not transpose. + """Test case where x < y and should not transpose.""" input_array = np.ma.array([[1, 2], [3, 4], [5, 6]]) result = transpose_if_xdim_less_than_ydim(input_array) np.testing.assert_array_equal(result, input_array) self.assertEqual(result.shape, (3, 2)) def test_square_array(self): - # test where y dim == x dim and should not transpose. + """Test case where y dim == x dim and should not transpose.""" input_array = np.ma.array([[1, 2], [3, 4]]) result = transpose_if_xdim_less_than_ydim(input_array) np.testing.assert_array_equal(result, input_array) self.assertEqual(result.shape, (2, 2)) def test_1d_array(self): - # Test case with a 1D array + """Test case with a 1D array""" input_array = np.ma.array([1, 2, 3]) with self.assertRaisesRegex(ValueError, 'variable must be 2 dimensional'): transpose_if_xdim_less_than_ydim(input_array) def test_3d_array(self): - # Test case with a 3D array + """Test case with a 3D array""" input_array = np.ma.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]) with self.assertRaisesRegex(ValueError, 'variable must be 2 dimensional'): transpose_if_xdim_less_than_ydim(input_array) def test_masked_array(self): - # Test case with a masked array + """Test case with a masked array""" input_array = np.ma.array( [[1, 2, 3], [4, 5, 6]], mask=[[True, False, False], [False, True, False]] ) From b707bd2454d5d9ae9ccb4a6e0ed6776610b23dce Mon Sep 17 00:00:00 2001 From: Joey Schultz Date: Wed, 23 Oct 2024 13:13:24 -0400 Subject: [PATCH 19/22] Remove TEMPO_O3TOT_L2_example.ipynb and add it to JIRA ticket instead. --- docs/TEMPO_O3TOT_L2_example.ipynb | 221 ------------------------------ 1 file changed, 221 deletions(-) delete mode 100644 docs/TEMPO_O3TOT_L2_example.ipynb diff --git a/docs/TEMPO_O3TOT_L2_example.ipynb b/docs/TEMPO_O3TOT_L2_example.ipynb deleted file mode 100644 index d4ec1df..0000000 --- a/docs/TEMPO_O3TOT_L2_example.ipynb +++ /dev/null @@ -1,221 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# TEMPO_O3TOT_L2 Harmony Tests" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Environment setup:\n", - "\n", - "This notebook assumes that it is being run in a local Python environment, configured using either `pyenv` or conda. Either can be used, but the dependencies will be installed via Pip. To install the required packages to run this notebook:\n", - "\n", - "```bash\n", - "$ pip install -r requirements.txt\n", - "```\n", - "\n", - "A `.netrc` file must also be located in the user's home directory." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Import Requirements:" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from os import makedirs, replace\n", - "from os.path import exists\n", - "from os.path import join as path_join\n", - "from shutil import rmtree\n", - "\n", - "from harmony import Client, Collection, Environment, Request\n", - "from harmony.harmony import ProcessingFailedException" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Set the Harmony Environment:" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "harmony_host_url = 'http://localhost:3000'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Identify Harmony environment (for easier reference):" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "host_environment = {\n", - " 'http://localhost:3000': Environment.LOCAL,\n", - " 'https://harmony.sit.earthdata.nasa.gov': Environment.SIT,\n", - " 'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n", - " 'https://harmony.earthdata.nasa.gov': Environment.PROD,\n", - "}\n", - "\n", - "\n", - "harmony_environment = host_environment.get(harmony_host_url)\n", - "\n", - "if harmony_environment is not None:\n", - " harmony_client = Client(env=harmony_environment)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### TEMPO_O3TOT_L2 Data:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "collection = Collection(id='C1270257471-EEDTEST')\n", - "granule_id = 'G1270257472-EEDTEST'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Create output directory to save test results:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "collection_short_name = 'TEMPO_O3TOT_L2'\n", - "output_directory = path_join('output_files', collection_short_name) + '/'\n", - "\n", - "if exists(output_directory):\n", - " rmtree(output_directory)\n", - "\n", - "makedirs(output_directory)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Utility functions for submitting Harmony Requests:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def submit_and_download(\n", - " harmony_client: Client, request: Request, output_file_name: str\n", - "):\n", - " \"\"\"Submit a Harmony request via a `harmony-py` client. Wait for the\n", - " Harmony job to finish, then download the results to the specified file\n", - " path.\n", - "\n", - " \"\"\"\n", - " downloaded_filename = None\n", - "\n", - " try:\n", - " job_id = harmony_client.submit(request)\n", - "\n", - " for filename in [\n", - " file_future.result()\n", - " for file_future in harmony_client.download_all(job_id, overwrite=True)\n", - " ]:\n", - "\n", - " print(f'Downloaded: {filename}')\n", - " downloaded_filename = filename\n", - "\n", - " if downloaded_filename is not None:\n", - " replace(downloaded_filename, output_file_name)\n", - " print(f'Saved output to: {output_file_name}')\n", - "\n", - " except ProcessingFailedException:\n", - " print_error('Harmony request failed to complete successfully.')\n", - "\n", - "\n", - "def print_error(error_string: str) -> str:\n", - " \"\"\"Print an error, with formatting for red text.\"\"\"\n", - " print(f'\\033[91m{error_string}\\033[0m')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Submitting Harmony Request:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "harmony_client = Client(env=Environment.LOCAL)\n", - "request = Request(\n", - " crs='EPSG:4326', interpolation='near', collection=collection, granule_id=granule_id\n", - ")\n", - "\n", - "swath_output_file_name = output_directory + f'TEMPO_O3TOT_L2_{granule_id}_output.h5'\n", - "submit_and_download(harmony_client, request, swath_output_file_name)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "localharmony", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.10" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From 6678d5a8f93fab5a79213d7ceb16a13538fb52c1 Mon Sep 17 00:00:00 2001 From: Joey Schultz Date: Fri, 25 Oct 2024 14:06:50 -0400 Subject: [PATCH 20/22] Simplify variable transposal, update effected typehints and unit tests --- CHANGELOG.md | 2 +- swath_projector/earthdata_varinfo_config.json | 2 +- swath_projector/swath_geometry.py | 16 ++++---- swath_projector/utilities.py | 38 +++++++++---------- tests/unit/test_swath_geometry.py | 12 +++--- tests/unit/test_utilities.py | 2 +- 6 files changed, 34 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3088ee..37602f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Changed - [[DAS-2216](https://bugs.earthdata.nasa.gov/browse/DAS-2216)] - The Swath Projector has been updated with quick fixes to add support for TEMPO level 2 data. + The Swath Projector has been updated with quick fixes to add support for TEMPO level 2 data. These changes include optional transposing of arrays based on dimension sizes and updates to the configuration file for TEMPO_O3TOT_L2 to correctly locate coordinate variables and exclude science variables with dimensions that do no match those of the coordinate variables. ## [v1.1.1] - 2024-09-16 diff --git a/swath_projector/earthdata_varinfo_config.json b/swath_projector/earthdata_varinfo_config.json index 25069d4..3a1132f 100644 --- a/swath_projector/earthdata_varinfo_config.json +++ b/swath_projector/earthdata_varinfo_config.json @@ -57,7 +57,7 @@ "Value": "/geolocation/latitude, /geolocation/longitude" } ], - "_Description": "TEMPO_O3TOT_L2 variables only contain basenames for coordinates, which are found in sibling hierarchical groups. This rule fully qualifies the paths to these coordinates." + "_Description": "TEMPO_O3TOT_L2 variables only contain basenames for coordinates, which are found in sibling hierarchical groups. This rule fully qualifies the paths to these coordinates. Some variables in these groups are excluded via 'ExcludedScienceVariables'" } ] } diff --git a/swath_projector/swath_geometry.py b/swath_projector/swath_geometry.py index e6da482..55890cc 100644 --- a/swath_projector/swath_geometry.py +++ b/swath_projector/swath_geometry.py @@ -13,7 +13,7 @@ def get_projected_resolution( - projection: Proj, longitudes: Variable, latitudes: Variable + projection: Proj, longitudes: np.ma.MaskedArray, latitudes: np.ma.MaskedArray ) -> Tuple[float]: """Find the resolution of the target grid in the projected coordinates, x and y. First the perimeter points are found. These are then projected @@ -40,7 +40,7 @@ def get_projected_resolution( def get_extents_from_perimeter( - projection: Proj, longitudes: Variable, latitudes: Variable + projection: Proj, longitudes: np.ma.MaskedArray, latitudes: np.ma.MaskedArray ) -> Tuple[float]: """Find the swath extents in the target CRS. First the perimeter points of unfilled valid pixels are found. These are then projected to the target @@ -59,19 +59,17 @@ def get_extents_from_perimeter( def get_projected_coordinates( coordinates_mask: np.ma.core.MaskedArray, projection: Proj, - longitudes: Variable, - latitudes: Variable, + longitudes: np.ma.MaskedArray, + latitudes: np.ma.MaskedArray, ) -> Tuple[np.ndarray]: """Get the required coordinate points projected in the target Coordinate Reference System (CRS). """ if len(longitudes.shape) == 1: - coordinates = get_all_coordinates(longitudes[:], latitudes[:], coordinates_mask) + coordinates = get_all_coordinates(longitudes, latitudes, coordinates_mask) else: - coordinates = get_perimeter_coordinates( - longitudes[:], latitudes[:], coordinates_mask - ) + coordinates = get_perimeter_coordinates(longitudes, latitudes, coordinates_mask) return reproject_coordinates(coordinates, projection) @@ -135,7 +133,7 @@ def get_absolute_resolution(polygon_area: float, n_pixels: int) -> float: def get_valid_coordinates_mask( - longitudes: Variable, latitudes: Variable + longitudes: np.ma.MaskedArray, latitudes: np.ma.MaskedArray ) -> np.ma.core.MaskedArray: """Get a `numpy` N-d array containing boolean values (0 or 1) indicating whether the elements of both longitude and latitude are valid at that diff --git a/swath_projector/utilities.py b/swath_projector/utilities.py index df67795..888bf55 100644 --- a/swath_projector/utilities.py +++ b/swath_projector/utilities.py @@ -44,23 +44,25 @@ def get_variable_values( return make_array_two_dimensional(variable[:]) elif 'time' in input_file.variables and 'time' in variable.dimensions: # Assumption: Array = (time, along-track, across-track) - return transpose_if_xdim_less_than_ydim( - variable[0][:].filled(fill_value=fill_value) + return transpose_if_xdim_less_than_ydim(variable[0][:]).filled( + fill_value=fill_value ) else: # Assumption: Array = (along-track, across-track) - return transpose_if_xdim_less_than_ydim( - variable[:].filled(fill_value=fill_value) + return transpose_if_xdim_less_than_ydim(variable[:]).filled( + fill_value=fill_value ) def get_coordinate_variable( dataset: Dataset, coordinates_tuple: Tuple[str], coordinate_substring -) -> Optional[Variable]: +) -> Optional[np.ma.MaskedArray]: """Search the coordinate dataset names for a match to the substring, which will be either "lat" or "lon". Return the corresponding variable - from the dataset. Only the base variable name is used, as the group - path may contain either of the strings as part of other words. + data from the dataset. Only the base variable name is used, as the group + path may contain either of the strings as part of other words. The + coordinate variables are transposed if the `along-track`dimension size is + less than the `across-track` dimension size. """ for coordinate in coordinates_tuple: @@ -69,9 +71,9 @@ def get_coordinate_variable( ): # QuickFix (DAS-2216) for short and wide swaths if dataset[coordinate].ndim == 1: - return dataset[coordinate] + return dataset[coordinate][:] - return transpose_if_xdim_less_than_ydim(dataset[coordinate]) + return transpose_if_xdim_less_than_ydim(dataset[coordinate][:]) raise MissingCoordinatesError(coordinates_tuple) @@ -228,21 +230,19 @@ def make_array_two_dimensional(one_dimensional_array: np.ndarray) -> np.ndarray: return np.expand_dims(one_dimensional_array, 1) -def transpose_if_xdim_less_than_ydim(variable: np.ndarray) -> np.ndarray: +def transpose_if_xdim_less_than_ydim( + variable_values: np.ma.MaskedArray, +) -> np.ma.MaskedArray: """Return transposed variable when variable is wider than tall. QuickFix (DAS-2216): We presume that a swath has more rows than columns and if that's not the case we transpose it so that it does. """ - if variable.ndim != 2: + if len(variable_values.shape) != 2: raise ValueError( - f'Input variable must be 2 dimensional, but got {variable.ndim} dimensions.' + f'Input variable must be 2 dimensional, but got {len(variable_values.shape)} dimensions.' ) - if variable.shape[0] < variable.shape[1]: - transposed_variable = np.ma.transpose(variable).copy() - if hasattr(variable, 'mask'): - transposed_variable.mask = np.ma.transpose(variable[:].mask) + if variable_values.shape[0] < variable_values.shape[1]: + return np.ma.transpose(variable_values).copy() - return transposed_variable - - return variable + return variable_values diff --git a/tests/unit/test_swath_geometry.py b/tests/unit/test_swath_geometry.py index 68bae25..896b380 100644 --- a/tests/unit/test_swath_geometry.py +++ b/tests/unit/test_swath_geometry.py @@ -65,8 +65,8 @@ def setUpClass(cls): def setUp(self): self.test_dataset = Dataset(self.test_path) - self.longitudes = self.test_dataset['lon'] - self.latitudes = self.test_dataset['lat'] + self.longitudes = self.test_dataset['lon'][:] + self.latitudes = self.test_dataset['lat'][:] def tearDown(self): self.test_dataset.close() @@ -101,8 +101,8 @@ def test_get_projected_resolution_1d(self): """Ensure the calculated one-dimensional resolution is correct.""" resolution = get_projected_resolution( self.geographic_projection, - self.test_dataset['lon_1d'], - self.test_dataset['lat_1d'], + self.test_dataset['lon_1d'][:], + self.test_dataset['lat_1d'][:], ) self.assertAlmostEqual(resolution, 5.0) @@ -167,9 +167,7 @@ def test_get_perimeter_coordinates(self): np.logical_not(valid_pixels), np.ones(self.longitudes.shape) ) - coordinates = get_perimeter_coordinates( - self.longitudes[:], self.latitudes[:], mask - ) + coordinates = get_perimeter_coordinates(self.longitudes, self.latitudes, mask) self.assertCountEqual(coordinates, expected_points) diff --git a/tests/unit/test_utilities.py b/tests/unit/test_utilities.py index b31e00f..0b78611 100644 --- a/tests/unit/test_utilities.py +++ b/tests/unit/test_utilities.py @@ -138,7 +138,7 @@ def test_get_coordinate_variables(self): dataset, coordinates_tuple, coordinate ) - self.assertIsInstance(coordinates, Variable) + self.assertIsInstance(coordinates, np.ma.MaskedArray) with self.subTest( 'Non existent coordinate variable "latitude" returns MissingCoordinatesError' From bde29b6f92898b8fe6001e054499059603810bfe Mon Sep 17 00:00:00 2001 From: Joey Schultz Date: Mon, 28 Oct 2024 11:54:38 -0400 Subject: [PATCH 21/22] Add get_rows_per_scan utility and associated unit tests --- CHANGELOG.md | 2 +- swath_projector/interpolation.py | 3 ++- swath_projector/utilities.py | 13 +++++++++++++ tests/unit/test_utilities.py | 15 +++++++++++++++ 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37602f1..c85e6c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Changed - [[DAS-2216](https://bugs.earthdata.nasa.gov/browse/DAS-2216)] - The Swath Projector has been updated with quick fixes to add support for TEMPO level 2 data. These changes include optional transposing of arrays based on dimension sizes and updates to the configuration file for TEMPO_O3TOT_L2 to correctly locate coordinate variables and exclude science variables with dimensions that do no match those of the coordinate variables. + The Swath Projector has been updated with quick fixes to add support for TEMPO level 2 data. These changes include optional transposing of arrays based on dimension sizes, addition of rows_per_scan parameter for ewa interpolation, and updates to the configuration file for TEMPO_O3TOT_L2 to correctly locate coordinate variables and exclude science variables with dimensions that do no match those of the coordinate variables. ## [v1.1.1] - 2024-09-16 diff --git a/swath_projector/interpolation.py b/swath_projector/interpolation.py index cfa5a63..9fc92e3 100644 --- a/swath_projector/interpolation.py +++ b/swath_projector/interpolation.py @@ -25,6 +25,7 @@ from swath_projector.utilities import ( create_coordinates_key, get_coordinate_variable, + get_rows_per_scan, get_scale_and_offset, get_variable_file_path, get_variable_numeric_fill_value, @@ -268,7 +269,7 @@ def get_ewa_results( ewa_information['target_area'], variable['values'], maximum_weight_mode=maximum_weight_mode, - rows_per_scan=2, # Added in QuickFix DAS-2216 to be fixed in DAS-2220 + rows_per_scan=get_rows_per_scan(variable['values'].shape[0]), ) if variable['fill_value'] is not None: diff --git a/swath_projector/utilities.py b/swath_projector/utilities.py index 888bf55..468a14c 100644 --- a/swath_projector/utilities.py +++ b/swath_projector/utilities.py @@ -246,3 +246,16 @@ def transpose_if_xdim_less_than_ydim( return np.ma.transpose(variable_values).copy() return variable_values + + +def get_rows_per_scan(total_rows: int) -> int: + """ + Finds the smallest divisor of the total number of rows. If no divisor is + found, return the total number of rows. + """ + if total_rows < 2: + return 1 + for row_number in range(2, int(total_rows**0.5) + 1): + if total_rows % row_number == 0: + return row_number + return total_rows diff --git a/tests/unit/test_utilities.py b/tests/unit/test_utilities.py index 0b78611..fe3eae3 100644 --- a/tests/unit/test_utilities.py +++ b/tests/unit/test_utilities.py @@ -10,6 +10,7 @@ construct_absolute_path, create_coordinates_key, get_coordinate_variable, + get_rows_per_scan, get_scale_and_offset, get_variable_file_path, get_variable_numeric_fill_value, @@ -411,3 +412,17 @@ def test_masked_array(self): result = transpose_if_xdim_less_than_ydim(input_array) np.testing.assert_array_equal(result, expected_output) np.testing.assert_array_equal(result.mask, expected_output.mask) + + +class TestGetRowsPerScan(TestCase): + def test_number_less_than_2(self): + self.assertEqual(get_rows_per_scan(1), 1) + + def test_even_composite_number(self): + self.assertEqual(get_rows_per_scan(4), 2) + + def test_odd_composite_number(self): + self.assertEqual(get_rows_per_scan(9), 3) + + def test_prime_number(self): + self.assertEqual(get_rows_per_scan(3), 3) From cd54588ad599b4ddebcaba3b824698b2d9aa3fdb Mon Sep 17 00:00:00 2001 From: Joey Schultz Date: Mon, 28 Oct 2024 15:53:23 -0400 Subject: [PATCH 22/22] Apply coordinates MetadataOverride to geolocation group --- swath_projector/earthdata_varinfo_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swath_projector/earthdata_varinfo_config.json b/swath_projector/earthdata_varinfo_config.json index 3a1132f..7af6c25 100644 --- a/swath_projector/earthdata_varinfo_config.json +++ b/swath_projector/earthdata_varinfo_config.json @@ -49,7 +49,7 @@ "Applicability": { "Mission": "TEMPO", "ShortNamePath": "TEMPO_O3TOT_L2", - "VariablePattern": "^/product/.*|^/support_data/.*" + "VariablePattern": "^/product/.*|^/support_data/.*|^/geolocation/(?!time$).*" }, "Attributes": [ {