From b74cc0eaeef32e638f5ca5581f7dedd20910750d Mon Sep 17 00:00:00 2001 From: Gerrit Holl Date: Wed, 18 Dec 2024 19:24:13 +0100 Subject: [PATCH 01/11] Combine low level moisture with a cloud mask Combine ESSL colorized low level moisture with a cloud mask. Work in progress. --- satpy/etc/composites/fci.yaml | 46 +++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/satpy/etc/composites/fci.yaml b/satpy/etc/composites/fci.yaml index 5dd812c73f..d33827de57 100644 --- a/satpy/etc/composites/fci.yaml +++ b/satpy/etc/composites/fci.yaml @@ -423,3 +423,49 @@ composites: - name: ir_38 modifiers: [nir_reflectance] standard_name: snow + + masked_colorised_low_level_moisture: + compositor: !!python/name:satpy.composites.MaskingCompositor + standard_name: essl_colorised_low_level_moisture + prerequisites: + - essl_colorized_low_level_moisture + - cloud_state + conditions: + - method: equal + value: Not processed (no or corrupt data) + transparency: 100 + - method: equal + value: Cloud free (no cloud, snow or ice) + transparency: 0 + - method: equal + value: Cloud contaminated (partial or semitransparent cloud) + transparency: 100 + - method: equal + value: Cloud filled (opaque cloud filled) + transparency: 100 + - method: equal + value: Dust contaminated + transparency: 100 + - method: equal + value: Dust filled (opaque) + transparency: 100 + - method: equal + value: Ash contaminated + transparency: 100 + - method: equal + value: Ash filled (opaque) + transparency: 100 + - method: equal + value: Snow or ice contaminated + transparency: 0 + - method: equal + value: Undefined + transparency: 0 + mode: RGBA + + colorised_low_level_moisture_with_vis06: + compositor: !!python/name:satpy.composites.BackgroundCompositor + standard_name: image_ready + prerequisites: + - masked_colorised_low_level_moisture + - dwd_vis06 From c423d21190cdb848f1b736cb52800913283b8265 Mon Sep 17 00:00:00 2001 From: Gerrit Holl Date: Tue, 7 Jan 2025 15:18:08 +0100 Subject: [PATCH 02/11] fix small mistakes in composite/enhancement definitions --- satpy/etc/composites/fci.yaml | 12 ++++++------ satpy/etc/enhancements/generic.yaml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/satpy/etc/composites/fci.yaml b/satpy/etc/composites/fci.yaml index d33827de57..0db888c6f2 100644 --- a/satpy/etc/composites/fci.yaml +++ b/satpy/etc/composites/fci.yaml @@ -424,9 +424,9 @@ composites: modifiers: [nir_reflectance] standard_name: snow - masked_colorised_low_level_moisture: + masked_colorized_low_level_moisture: compositor: !!python/name:satpy.composites.MaskingCompositor - standard_name: essl_colorised_low_level_moisture + standard_name: essl_colorized_low_level_moisture prerequisites: - essl_colorized_low_level_moisture - cloud_state @@ -461,11 +461,11 @@ composites: - method: equal value: Undefined transparency: 0 - mode: RGBA + mode: LA - colorised_low_level_moisture_with_vis06: + colorized_low_level_moisture_with_vis06: compositor: !!python/name:satpy.composites.BackgroundCompositor standard_name: image_ready prerequisites: - - masked_colorised_low_level_moisture - - dwd_vis06 + - masked_colorized_low_level_moisture + - vis_06 diff --git a/satpy/etc/enhancements/generic.yaml b/satpy/etc/enhancements/generic.yaml index cdfb7851ad..8f5b4f0760 100644 --- a/satpy/etc/enhancements/generic.yaml +++ b/satpy/etc/enhancements/generic.yaml @@ -1241,7 +1241,7 @@ enhancements: operations: [] essl_colorized_low_level_moisture: - name: essl_colorized_low_level_moisture + standard_name: essl_colorized_low_level_moisture operations: - name: essl_moisture method: !!python/name:satpy.enhancements.atmosphere.essl_moisture From a470510172bd291191c537609963700648176250 Mon Sep 17 00:00:00 2001 From: Gerrit Holl Date: Tue, 7 Jan 2025 17:17:59 +0100 Subject: [PATCH 03/11] for some reason masked needs its own enhancement If I don't give its own enhancement to the masked colorised llm, either the masked or the unmasked version doesn't find its enhancement. --- satpy/etc/composites/fci.yaml | 2 +- satpy/etc/enhancements/generic.yaml | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/satpy/etc/composites/fci.yaml b/satpy/etc/composites/fci.yaml index 0db888c6f2..9c6871d3d6 100644 --- a/satpy/etc/composites/fci.yaml +++ b/satpy/etc/composites/fci.yaml @@ -426,7 +426,7 @@ composites: masked_colorized_low_level_moisture: compositor: !!python/name:satpy.composites.MaskingCompositor - standard_name: essl_colorized_low_level_moisture + standard_name: masked_essl_colorized_low_level_moisture prerequisites: - essl_colorized_low_level_moisture - cloud_state diff --git a/satpy/etc/enhancements/generic.yaml b/satpy/etc/enhancements/generic.yaml index 38592c232f..486d2e1d85 100644 --- a/satpy/etc/enhancements/generic.yaml +++ b/satpy/etc/enhancements/generic.yaml @@ -1241,8 +1241,9 @@ enhancements: operations: [] essl_colorized_low_level_moisture: - standard_name: essl_colorized_low_level_moisture - operations: + # this enhancement is only found if using name but not standard_name + name: essl_colorized_low_level_moisture + operations: &masked_llm - name: colorize method: !!python/name:satpy.enhancements.colorize kwargs: @@ -1352,6 +1353,11 @@ enhancements: - [255, 249, 183] - [255, 255, 191] + masked_essl_colorized_low_level_moisture: + # this enhancement is only found if using standard_name but not name + standard_name: masked_essl_colorized_low_level_moisture + operations: *masked_llm + rocket_plume: standard_name: rocket_plume operations: From b829d96841a9697a242474c771b14b73390fbf70 Mon Sep 17 00:00:00 2001 From: Gerrit Holl Date: Wed, 8 Jan 2025 16:12:35 +0100 Subject: [PATCH 04/11] add an effective solar pathlength corrector --- satpy/etc/composites/fci.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/satpy/etc/composites/fci.yaml b/satpy/etc/composites/fci.yaml index 9c6871d3d6..81468d533b 100644 --- a/satpy/etc/composites/fci.yaml +++ b/satpy/etc/composites/fci.yaml @@ -468,4 +468,12 @@ composites: standard_name: image_ready prerequisites: - masked_colorized_low_level_moisture - - vis_06 + - name: vis_06 + modifiers: [effective_solar_pathlength_corrected] + + colorized_low_level_moisture_with_ir105: + compositor: !!python/name:satpy.composites.BackgroundCompositor + standard_name: image_ready + prerequisites: + - masked_colorized_low_level_moisture + - night_ir105 From 74f22c362c74a3602441249172827e967a246b45 Mon Sep 17 00:00:00 2001 From: Gerrit Holl Date: Tue, 21 Jan 2025 17:20:13 +0100 Subject: [PATCH 05/11] create reference images with multiple readers For the create reference script, add the ability to use multiple readers. --- utils/create_reference.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/utils/create_reference.py b/utils/create_reference.py index 04bffdd9a3..310f4f0201 100644 --- a/utils/create_reference.py +++ b/utils/create_reference.py @@ -45,7 +45,11 @@ def generate_images(props): filenames = (props.basedir / "satellite_data" / props.satellite / props.case).glob("*") - scn = Scene(reader=props.reader, filenames=filenames) + if "," in props.reader: + reader = props.reader.split(",") + else: + reader = props.reader + scn = Scene(reader=reader, filenames=filenames) scn.load(props.composites) if props.area == "native": @@ -74,7 +78,7 @@ def get_parser(): parser.add_argument( "reader", action="store", type=str, - help="Reader name.") + help="Reader name. Multiple readers (if needed) can be comma-seperated.") parser.add_argument( "case", help="case to generate", type=str) From 82b2cebb616c8a7ce4e232bb84f03de9e7437939 Mon Sep 17 00:00:00 2001 From: Gerrit Holl Date: Fri, 24 Jan 2025 09:47:32 +0100 Subject: [PATCH 06/11] improve behave tests Improve behave tests. Allow to have multiple readers. When clipping, use the configuration option, because reader_kwargs with multiple readers is less practical / does not work very well. --- satpy/tests/behave/features/image_comparison.feature | 1 + satpy/tests/behave/features/steps/image_comparison.py | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/satpy/tests/behave/features/image_comparison.feature b/satpy/tests/behave/features/image_comparison.feature index 686062462c..44e249d189 100755 --- a/satpy/tests/behave/features/image_comparison.feature +++ b/satpy/tests/behave/features/image_comparison.feature @@ -10,6 +10,7 @@ Feature: Image Comparison |Meteosat-12 | scan_night | cloudtop | fci_l1c_nc | sve | True | |Meteosat-12 | scan_night | night_microphysics | fci_l1c_nc | sve | True | |Meteosat-12 | mali_day | essl_colorized_low_level_moisture | fci_l1c_nc | mali | False | + |Meteosat-12 | spain_day | colorized_low_level_moisture_with_vis06 | fci_l1c_nc,fci_l2_nc | spain | False | |GOES17 | americas_night | airmass | abi_l1b | null | null | |GOES16 | americas_night | airmass | abi_l1b | null | null | |GOES16 | americas_night | ash | abi_l1b | null | null | diff --git a/satpy/tests/behave/features/steps/image_comparison.py b/satpy/tests/behave/features/steps/image_comparison.py index 5e7135bc53..64a4415377 100644 --- a/satpy/tests/behave/features/steps/image_comparison.py +++ b/satpy/tests/behave/features/steps/image_comparison.py @@ -27,6 +27,7 @@ import numpy as np from behave import given, then, when +import satpy from satpy import Scene ext_data_path = "/app/ext_data" @@ -75,12 +76,12 @@ def step_when_generate_image(context, composite, satellite, case, reader, area, # Get the list of satellite files to open filenames = glob(f"{ext_data_path}/satellite_data/{satellite}/{case}/*.nc") - reader_kwargs = {} - if clip != "null": - reader_kwargs["clip_negative_radiances"] = clip - scn = Scene(reader=reader, filenames=filenames, reader_kwargs=reader_kwargs) - scn.load([composite]) + if "," in reader: + reader = reader.split(",") + with satpy.config.set({"readers.clip_negative_radiances": False if clip == "null" else clip}): + scn = Scene(reader=reader, filenames=filenames) + scn.load([composite]) if area == "null": ls = scn From 6a7a78a9fc927e6d5524f94a66771d621b57ca35 Mon Sep 17 00:00:00 2001 From: Gerrit Holl Date: Fri, 24 Jan 2025 10:41:06 +0100 Subject: [PATCH 07/11] Workaround for satpy issue 1913 --- satpy/etc/composites/visir.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/satpy/etc/composites/visir.yaml b/satpy/etc/composites/visir.yaml index fa774e26da..7b6f6d4243 100644 --- a/satpy/etc/composites/visir.yaml +++ b/satpy/etc/composites/visir.yaml @@ -603,9 +603,11 @@ composites: European Severe Storms Laboratory (ESSL). For a color version, see essl_colorized_low_level_moisture. compositor: !!python/name:satpy.composites.RatioCompositor - prerequisites: - - wavelength: 0.905 - - wavelength: 0.86 + prerequisites: &llm + - wavelength: 0.905 # workaround for https://github.com/pytroll/satpy/issues/1913 + calibration: reflectance + - wavelength: 0.86 + calibration: reflectance standard_name: essl_low_level_moisture day_essl_low_level_moisture: @@ -625,9 +627,7 @@ composites: European Severe Storms Laboratory (ESSL). The colorization is still under development and may be subject to change. compositor: !!python/name:satpy.composites.RatioCompositor - prerequisites: - - wavelength: 0.905 - - wavelength: 0.86 + prerequisites: *llm standard_name: essl_colorized_low_level_moisture day_essl_colorized_low_level_moisture: From 310f6d8f9ee29e388087933e7bed257cac2ea574 Mon Sep 17 00:00:00 2001 From: Gerrit Holl Date: Fri, 24 Jan 2025 11:00:54 +0100 Subject: [PATCH 08/11] add calibration to fake reader channels Add calibration to the fake reader channels matching 0.86 and 0.91, so that the workaround added in 6a7a78a9fc927e6d5524f94a66771d621b57ca35 does not change the total number of expected datasets in test_load.py --- satpy/tests/etc/readers/fake1.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/satpy/tests/etc/readers/fake1.yaml b/satpy/tests/etc/readers/fake1.yaml index b34c7e1b34..746c2ae04c 100644 --- a/satpy/tests/etc/readers/fake1.yaml +++ b/satpy/tests/etc/readers/fake1.yaml @@ -60,6 +60,7 @@ datasets: ds8: name: ds8 wavelength: [0.7, 0.8, 0.9] + calibration: "reflectance" file_type: fake_file1 coordinates: [lons, lats] ds9_fail_load: @@ -70,6 +71,7 @@ datasets: ds10: name: ds10 wavelength: [0.75, 0.85, 0.95] + calibration: "reflectance" file_type: fake_file1 coordinates: [lons, lats] ds11: From 2e48efeb1096c1145f84ba2a0e1589c874a2f029 Mon Sep 17 00:00:00 2001 From: Gerrit Holl Date: Fri, 24 Jan 2025 11:48:36 +0100 Subject: [PATCH 09/11] Use nearest for llm with vis06 Use nearest neighbour when combining with cloud mask --- .../behave/features/image_comparison.feature | 20 +++++++++---------- .../behave/features/steps/image_comparison.py | 6 +++--- utils/create_reference.py | 4 +++- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/satpy/tests/behave/features/image_comparison.feature b/satpy/tests/behave/features/image_comparison.feature index 44e249d189..5e97e08b23 100755 --- a/satpy/tests/behave/features/image_comparison.feature +++ b/satpy/tests/behave/features/image_comparison.feature @@ -2,16 +2,16 @@ Feature: Image Comparison Scenario Outline: Compare generated image with reference image Given I have a reference image file from resampled to - When I generate a new image file from case with for with clipping + When I generate a new image file from case with for resampling with with clipping Then the generated image should be the same as the reference image Examples: - |satellite | case | composite | reader | area | clip | - |Meteosat-12 | scan_night | cloudtop | fci_l1c_nc | sve | True | - |Meteosat-12 | scan_night | night_microphysics | fci_l1c_nc | sve | True | - |Meteosat-12 | mali_day | essl_colorized_low_level_moisture | fci_l1c_nc | mali | False | - |Meteosat-12 | spain_day | colorized_low_level_moisture_with_vis06 | fci_l1c_nc,fci_l2_nc | spain | False | - |GOES17 | americas_night | airmass | abi_l1b | null | null | - |GOES16 | americas_night | airmass | abi_l1b | null | null | - |GOES16 | americas_night | ash | abi_l1b | null | null | - |GOES17 | americas_night | ash | abi_l1b | null | null | + |satellite | case | composite | reader | area | resampler | clip | + |Meteosat-12 | scan_night | cloudtop | fci_l1c_nc | sve | gradient_search | True | + |Meteosat-12 | scan_night | night_microphysics | fci_l1c_nc | sve | gradient_search | True | + |Meteosat-12 | mali_day | essl_colorized_low_level_moisture | fci_l1c_nc | mali | gradient_search | False | + |Meteosat-12 | spain_day | colorized_low_level_moisture_with_vis06 | fci_l1c_nc,fci_l2_nc | nearest | spain | False | + |GOES17 | americas_night | airmass | abi_l1b | null | null | null | + |GOES16 | americas_night | airmass | abi_l1b | null | null | null | + |GOES16 | americas_night | ash | abi_l1b | null | null | null | + |GOES17 | americas_night | ash | abi_l1b | null | null | null | diff --git a/satpy/tests/behave/features/steps/image_comparison.py b/satpy/tests/behave/features/steps/image_comparison.py index 64a4415377..205a7f72b0 100644 --- a/satpy/tests/behave/features/steps/image_comparison.py +++ b/satpy/tests/behave/features/steps/image_comparison.py @@ -65,8 +65,8 @@ def step_given_reference_image(context, composite, satellite, area): @when("I generate a new {composite} image file from {satellite} case {case} " - "with {reader} for {area} with clipping {clip}") -def step_when_generate_image(context, composite, satellite, case, reader, area, clip): + "with {reader} for {area} resampling with {resampler} with clipping {clip}") +def step_when_generate_image(context, composite, satellite, case, reader, area, resampler,clip): """Generate test images.""" os.environ["OMP_NUM_THREADS"] = os.environ["MKL_NUM_THREADS"] = "2" os.environ["PYTROLL_CHUNK_SIZE"] = "1024" @@ -86,7 +86,7 @@ def step_when_generate_image(context, composite, satellite, case, reader, area, if area == "null": ls = scn else: - ls = scn.resample(area, resampler="gradient_search") + ls = scn.resample(area, resampler=resampler) # Save the generated image in the generated folder generated_image_path = os.path.join(context.test_results_dir, "generated", diff --git a/utils/create_reference.py b/utils/create_reference.py index 310f4f0201..6a63e5c0da 100644 --- a/utils/create_reference.py +++ b/utils/create_reference.py @@ -47,15 +47,17 @@ def generate_images(props): if "," in props.reader: reader = props.reader.split(",") + resampler = "nearest" # use nearest when combining with cloud mask else: reader = props.reader + resampler = "gradient_search" scn = Scene(reader=reader, filenames=filenames) scn.load(props.composites) if props.area == "native": ls = scn.resample(resampler="native") elif props.area is not None: - ls = scn.resample(props.area, resampler="gradient_search") + ls = scn.resample(props.area, resampler=resampler) else: ls = scn From b62a94022b9a8439cb2cbeef9a8c7fccb8a0306e Mon Sep 17 00:00:00 2001 From: Gerrit Holl Date: Fri, 24 Jan 2025 12:07:11 +0100 Subject: [PATCH 10/11] Fix mistake in feature definition The area and the resampler were swapped for one case in the feature definition. Fix this. --- satpy/tests/behave/features/image_comparison.feature | 2 +- satpy/tests/behave/features/steps/image_comparison.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/tests/behave/features/image_comparison.feature b/satpy/tests/behave/features/image_comparison.feature index 5e97e08b23..5fe2e1d383 100755 --- a/satpy/tests/behave/features/image_comparison.feature +++ b/satpy/tests/behave/features/image_comparison.feature @@ -10,7 +10,7 @@ Feature: Image Comparison |Meteosat-12 | scan_night | cloudtop | fci_l1c_nc | sve | gradient_search | True | |Meteosat-12 | scan_night | night_microphysics | fci_l1c_nc | sve | gradient_search | True | |Meteosat-12 | mali_day | essl_colorized_low_level_moisture | fci_l1c_nc | mali | gradient_search | False | - |Meteosat-12 | spain_day | colorized_low_level_moisture_with_vis06 | fci_l1c_nc,fci_l2_nc | nearest | spain | False | + |Meteosat-12 | spain_day | colorized_low_level_moisture_with_vis06 | fci_l1c_nc,fci_l2_nc | spain | nearest | False | |GOES17 | americas_night | airmass | abi_l1b | null | null | null | |GOES16 | americas_night | airmass | abi_l1b | null | null | null | |GOES16 | americas_night | ash | abi_l1b | null | null | null | diff --git a/satpy/tests/behave/features/steps/image_comparison.py b/satpy/tests/behave/features/steps/image_comparison.py index 205a7f72b0..fccd03650c 100644 --- a/satpy/tests/behave/features/steps/image_comparison.py +++ b/satpy/tests/behave/features/steps/image_comparison.py @@ -66,7 +66,7 @@ def step_given_reference_image(context, composite, satellite, area): @when("I generate a new {composite} image file from {satellite} case {case} " "with {reader} for {area} resampling with {resampler} with clipping {clip}") -def step_when_generate_image(context, composite, satellite, case, reader, area, resampler,clip): +def step_when_generate_image(context, composite, satellite, case, reader, area, resampler, clip): """Generate test images.""" os.environ["OMP_NUM_THREADS"] = os.environ["MKL_NUM_THREADS"] = "2" os.environ["PYTROLL_CHUNK_SIZE"] = "1024" From 480a58b6359ad42e6c36cf09032ee85bf2bd57a0 Mon Sep 17 00:00:00 2001 From: Gerrit Holl Date: Fri, 24 Jan 2025 12:36:12 +0100 Subject: [PATCH 11/11] Document new composites and add credit to ESSL. Document the composites combining low level moisture with a cloud mask, and add credit to ESSL for the colours. --- satpy/etc/composites/fci.yaml | 20 ++++++++++++++++++++ satpy/etc/enhancements/generic.yaml | 1 + 2 files changed, 21 insertions(+) diff --git a/satpy/etc/composites/fci.yaml b/satpy/etc/composites/fci.yaml index 81468d533b..8d88f795f9 100644 --- a/satpy/etc/composites/fci.yaml +++ b/satpy/etc/composites/fci.yaml @@ -425,6 +425,12 @@ composites: standard_name: snow masked_colorized_low_level_moisture: + description: > + Like essl_colorized_low_level_moisture, but with clouds masked out + according to the EUMETSAT FCI L2 CLM product. Note that due to the + categorical nature of the mask, resampling this composite should + only be done with nearest neighbour. The colormap for the cloudfree + part has been developed by the European Severe Storms Laboratory (ESSL). compositor: !!python/name:satpy.composites.MaskingCompositor standard_name: masked_essl_colorized_low_level_moisture prerequisites: @@ -464,6 +470,13 @@ composites: mode: LA colorized_low_level_moisture_with_vis06: + description: > + Like essl_colorized_low_level_moisture, but with clouds shown according + to the vis_06 channel (enhanced with effective_solar_pathlength_corrected). + Note that due to the categorical nature of the mask, resampling this + composite should only be done with nearest neighbour. The colormap + for the cloudfree part has been developed by the European Severe Storms + Laboratory (ESSL) compositor: !!python/name:satpy.composites.BackgroundCompositor standard_name: image_ready prerequisites: @@ -472,6 +485,13 @@ composites: modifiers: [effective_solar_pathlength_corrected] colorized_low_level_moisture_with_ir105: + description: > + Like essl_colorized_low_level_moisture, but with clouds shown according + to the in_105 channel (inverted so clouds are white). + Note that due to the categorical nature of the mask, resampling this + composite should only be done with nearest neighbour. The colormap + for the cloudfree part has been developed by the European Severe Storms + Laboratory (ESSL). compositor: !!python/name:satpy.composites.BackgroundCompositor standard_name: image_ready prerequisites: diff --git a/satpy/etc/enhancements/generic.yaml b/satpy/etc/enhancements/generic.yaml index 7abf83bcb5..47e21a5acf 100644 --- a/satpy/etc/enhancements/generic.yaml +++ b/satpy/etc/enhancements/generic.yaml @@ -1242,6 +1242,7 @@ enhancements: essl_colorized_low_level_moisture: # this enhancement is only found if using name but not standard_name + # The colormap was developed by the European Severe Storms Laboratory (ESSL). name: essl_colorized_low_level_moisture operations: &masked_llm - name: colorize