Mhs/das 2108/handle no data (#6)

.pre-commit-config.yaml

@@ -13,7 +13,7 @@ repos:
     rev: v0.3.7
     hooks:
       - id: ruff
-        args: ["--fix", "--show-fixes"]
+        args: ["--fix", "--show-fixes", "--select", "I"]
   - repo: https://github.com/psf/black-pre-commit-mirror
     rev: 24.4.0
     hooks:

CHANGELOG.md

@@ -1,3 +1,9 @@
+## v1.0.2
+### 2024-04-05
+
+This version of HyBIG correctly handles missing/bad input data marked by _FillValue or NoData.
+Anytime a bad value occurs in the input raster, the output png image will set to transparent.
+
 ## v1.0.1
 ### 2024-04-05
 

docker/service_version.txt

@@ -1 +1 @@
-1.0.1
+1.0.2

harmony_browse_image_generator/browse.py

@@ -13,7 +13,7 @@
 from harmony.message import Source as HarmonySource
 from matplotlib.cm import ScalarMappable
 from matplotlib.colors import Normalize
-from numpy import around, concatenate, ndarray
+from numpy import ndarray
 from osgeo_utils.auxiliary.color_palette import ColorPalette
 from PIL import Image
 from rasterio.io import DatasetReader
@@ -25,9 +25,12 @@
 from harmony_browse_image_generator.color_utility import (
     NODATA_IDX,
     NODATA_RGBA,
+    OPAQUE,
+    TRANSPARENT,
     TRANSPARENT_IDX,
     TRANSPARENT_RGBA,
     get_color_palette,
+    remove_alpha,
 )
 from harmony_browse_image_generator.exceptions import HyBIGError
 from harmony_browse_image_generator.sizes import (
@@ -113,27 +116,39 @@ def create_browse_imagery(
 def convert_mulitband_to_raster(data_array: DataArray) -> ndarray:
     """Convert multiband to a raster image.
 
-    Reads the three/four bands from the file, then normalizes them to the range
+    Reads the three or four bands from the file, then normalizes them to the range
     0 to 255. This assumes the input image is already in RGB or RGBA format and
     just ensures that the output is 8bit.
 
     """
+    if data_array.rio.count not in [3, 4]:
+        raise HyBIGError(
+            f'Cannot create image from {data_array.rio.count} band image. '
+            'Expecting 3 or 4 bands.'
+        )
+
     bands = data_array.to_numpy()
-    norm = Normalize()
 
-    if data_array.rio.count == 3:
-        norm.autoscale(bands)
-        raster = around(norm(bands) * 255.0).astype('uint8')
+    # Create an alpha layer where input NaN values are transparent.
+    nan_mask = np.isnan(bands).any(axis=0)
+    nan_alpha = np.where(nan_mask, TRANSPARENT, OPAQUE)
 
-    elif data_array.rio.count == 4:
-        norm.autoscale(bands[0:3, :, :])
-        partial_raster = around(norm(bands[0:3, :, :]) * 255.0).astype('uint8')
-        raster = concatenate([partial_raster.data, bands[3:4, :, :]])
+    # grab any existing alpha layer
+    bands, image_alpha = remove_alpha(bands)
+
+    norm = Normalize(vmin=np.nanmin(bands), vmax=np.nanmax(bands))
+    raster = np.nan_to_num(np.around(norm(bands) * 255.0), copy=False, nan=0.0).astype(
+        'uint8'
+    )
 
+    if image_alpha is not None:
+        # merge nan alpha with the image alpha prefering transparency to
+        # opaqueness.
+        alpha = np.minimum(nan_alpha, image_alpha)
     else:
-        raise HyBIGError(f'Cannot create image from {data_array.rio.count} band image')
+        alpha = nan_alpha
 
-    return raster
+    return np.concatenate((raster, alpha[None, ...]), axis=0)
 
 
 def convert_singleband_to_raster(
@@ -153,15 +168,16 @@ def convert_gray_1band_to_raster(data_array: DataArray) -> ndarray:
     """Convert a 1-band raster without a color association."""
     band = data_array[0, :, :]
     cmap = matplotlib.colormaps['Greys_r']
-    norm = Normalize()
-    norm.autoscale(band)
+    cmap.set_bad(TRANSPARENT_RGBA)
+    norm = Normalize(vmin=np.nanmin(band), vmax=np.nanmax(band))
     scalar_map = ScalarMappable(cmap=cmap, norm=norm)
 
     rgba_image = np.zeros((*band.shape, 4), dtype='uint8')
     for row_no in range(band.shape[0]):
         rgba_image_slice = scalar_map.to_rgba(band[row_no, :], bytes=True)
         rgba_image[row_no, :, :] = rgba_image_slice
-    return reshape_as_raster(rgba_image[..., 0:3])
+
+    return reshape_as_raster(rgba_image)
 
 
 def convert_paletted_1band_to_raster(
@@ -182,7 +198,7 @@ def convert_paletted_1band_to_raster(
         levels, scaled_colors, extend='max'
     )
 
-    # handle no data values
+    # handle palette no data value
     if palette.ndv is not None:
         nodata_colors = palette.color_to_color_entry(palette.ndv, with_alpha=True)
         cmap.set_bad(
@@ -222,13 +238,6 @@ def prepare_raster_for_writing(
     return palettize_raster(raster)
 
 
-def remove_alpha(raster: ndarray) -> tuple[ndarray, ndarray | None]:
-    """Pull raster array off of input if it exists."""
-    if raster.shape[0] == 4:
-        return raster[0:3, :, :], raster[3, :, :]
-    return raster, None
-
-
 def palettize_raster(raster: ndarray) -> tuple[ndarray, dict]:
     """convert an RGB or RGBA image into a 1band image and palette.
 
@@ -262,10 +271,9 @@ def add_alpha(
 ) -> tuple[ndarray, dict]:
     """If the input data had alpha values, manually set the quantized_image
     index to the transparent index in those places."""
-    max_alpha = 255
-    if alpha is not None and np.any(alpha != max_alpha):
+    if alpha is not None and np.any(alpha != OPAQUE):
         # Set any alpha to the transparent index value
-        quantized_array = np.where(alpha != max_alpha, TRANSPARENT_IDX, quantized_array)
+        quantized_array = np.where(alpha != OPAQUE, TRANSPARENT_IDX, quantized_array)
         color_map[TRANSPARENT_IDX] = TRANSPARENT_RGBA
     return quantized_array, color_map
 

harmony_browse_image_generator/color_utility.py

@@ -5,6 +5,9 @@
 
 """
 
+from typing import TYPE_CHECKING
+
+import numpy as np
 import requests
 from harmony.message import Source as HarmonySource
 from osgeo_utils.auxiliary.color_palette import ColorPalette
@@ -18,6 +21,8 @@
 
 # Constants for output PNG images
 # Applied to transparent pixels where alpha < 255
+TRANSPARENT = np.uint8(0)
+OPAQUE = np.uint8(255)
 TRANSPARENT_RGBA = (0, 0, 0, 0)
 TRANSPARENT_IDX = 254
 
@@ -26,6 +31,13 @@
 NODATA_IDX = 255
 
 
+def remove_alpha(raster: np.ndarray) -> tuple[np.ndarray, np.ndarray, None]:
+    """remove alpha layer when it exists."""
+    if raster.shape[0] == 4:
+        return raster[0:3, :, :], raster[3, :, :]
+    return raster, None
+
+
 def palette_from_remote_colortable(url: str) -> ColorPalette:
     """Return a gdal ColorPalette from a remote colortable."""
     response = requests.get(url, timeout=10)

tests/unit/test_browse.py

@@ -33,6 +33,8 @@
     validate_file_type,
 )
 from harmony_browse_image_generator.color_utility import (
+    OPAQUE,
+    TRANSPARENT,
     convert_colormap_to_palette,
     get_color_palette,
     palette_from_remote_colortable,
@@ -298,39 +300,46 @@ def test_create_browse_imagery_with_mocks(
         )
 
     def test_convert_singleband_to_raster_without_colortable(self):
-        ds = MagicMock(DataArray)
-        ds.__getitem__.return_value = self.data
+        """Tests convert_gray_1band_to_raster."""
+
+        return_data = np.copy(self.data).astype('float64')
+        return_data[0][1] = np.nan
+        ds = DataArray(return_data).expand_dims('band')
 
         expected_raster = np.array(
             [
                 [
-                    [0, 104, 198, 255],
+                    [0, 0, 198, 255],
                     [0, 104, 198, 255],
                     [0, 104, 198, 255],
                     [0, 104, 198, 255],
                 ],
                 [
-                    [0, 104, 198, 255],
+                    [0, 0, 198, 255],
                     [0, 104, 198, 255],
                     [0, 104, 198, 255],
                     [0, 104, 198, 255],
                 ],
                 [
+                    [0, 0, 198, 255],
                     [0, 104, 198, 255],
                     [0, 104, 198, 255],
                     [0, 104, 198, 255],
-                    [0, 104, 198, 255],
+                ],
+                [
+                    [255, 0, 255, 255],
+                    [255, 255, 255, 255],
+                    [255, 255, 255, 255],
+                    [255, 255, 255, 255],
                 ],
             ],
             dtype='uint8',
         )
-
         actual_raster = convert_singleband_to_raster(ds, None)
         assert_array_equal(expected_raster, actual_raster)
 
     def test_convert_singleband_to_raster_with_colormap(self):
-        ds = MagicMock(DataArray)
-        ds.__getitem__.return_value = self.data
+        ds = DataArray(self.data).expand_dims('band')
 
         expected_raster = np.array(
             [
@@ -367,13 +376,11 @@ def test_convert_singleband_to_raster_with_colormap(self):
         assert_array_equal(expected_raster, actual_raster)
 
     def test_convert_singleband_to_raster_with_colormap_and_bad_data(self):
-        ds = MagicMock(DataArray)
         data_array = np.array(self.data, dtype='float')
         data_array[0, 0] = np.nan
+        ds = DataArray(data_array).expand_dims('band')
         nv_color = (10, 20, 30, 40)
 
-        ds.__getitem__.return_value = data_array
-
         # Read the image down: red, yellow, green, blue
         expected_raster = np.array(
             [
@@ -412,9 +419,14 @@ def test_convert_singleband_to_raster_with_colormap_and_bad_data(self):
         assert_array_equal(expected_raster, actual_raster)
 
     def test_convert_3_multiband_to_raster(self):
-        ds = Mock(DataArray)
-        ds.rio.count = 3
-        ds.to_numpy.return_value = np.stack([self.data, self.data, self.data])
+
+        bad_data = np.copy(self.data).astype('float64')
+        bad_data[1][1] = np.nan
+        bad_data[1][2] = np.nan
+        ds = DataArray(
+            np.stack([self.data, bad_data, self.data]),
+            dims=('band', 'y', 'x'),
+        )
 
         expected_raster = np.array(
             [
@@ -426,7 +438,7 @@ def test_convert_3_multiband_to_raster(self):
                 ],
                 [
                     [0, 85, 170, 255],
-                    [0, 85, 170, 255],
+                    [0, 0, 0, 255],
                     [0, 85, 170, 255],
                     [0, 85, 170, 255],
                 ],
@@ -436,6 +448,12 @@ def test_convert_3_multiband_to_raster(self):
                     [0, 85, 170, 255],
                     [0, 85, 170, 255],
                 ],
+                [
+                    [OPAQUE, OPAQUE, OPAQUE, OPAQUE],
+                    [OPAQUE, TRANSPARENT, TRANSPARENT, OPAQUE],
+                    [OPAQUE, OPAQUE, OPAQUE, OPAQUE],
+                    [OPAQUE, OPAQUE, OPAQUE, OPAQUE],
+                ],
             ],
             dtype='uint8',
         )
@@ -444,16 +462,27 @@ def test_convert_3_multiband_to_raster(self):
         assert_array_equal(expected_raster, actual_raster.data)
 
     def test_convert_4_multiband_to_raster(self):
+        """Input data has NaN _fillValue match in the red layer at [1,1]
+        and alpha channel also exists with a single transparent value at [0,0]
+
+        See that the expected output has transformed the missing data [nan]
+        into fully transparent at [1,1] and retained the transparent value of 1
+        at [0,0]
+
+        """
         ds = Mock(DataArray)
+        bad_data = np.copy(self.data).astype('float64')
+        bad_data[1, 1] = np.nan
+
         alpha = np.ones_like(self.data) * 255
         alpha[0, 0] = 1
         ds.rio.count = 4
-        ds.to_numpy.return_value = np.stack([self.data, self.data, self.data, alpha])
+        ds.to_numpy.return_value = np.stack([bad_data, self.data, self.data, alpha])
         expected_raster = np.array(
             [
                 [
                     [0, 85, 170, 255],
-                    [0, 85, 170, 255],
+                    [0, 0, 170, 255],
                     [0, 85, 170, 255],
                     [0, 85, 170, 255],
                 ],
@@ -471,7 +500,7 @@ def test_convert_4_multiband_to_raster(self):
                 ],
                 [
                     [1, 255, 255, 255],
-                    [255, 255, 255, 255],
+                    [255, 0, 255, 255],
                     [255, 255, 255, 255],
                     [255, 255, 255, 255],
                 ],
@@ -493,7 +522,8 @@ def test_convert_5_multiband_to_raster(self):
             convert_mulitband_to_raster(ds)
 
         self.assertEqual(
-            excepted.exception.message, 'Cannot create image from 5 band image'
+            excepted.exception.message,
+            'Cannot create image from 5 band image. Expecting 3 or 4 bands.',
         )
 
     def test_prepare_raster_for_writing_jpeg_3band(self):

tests/utilities.py

@@ -6,7 +6,6 @@
 from harmony.util import bbox_to_geometry
 from pystac import Asset, Catalog, Item
 
-
 Granule = namedtuple('Granule', ['url', 'media_type', 'roles'])