From 90c13d4f25353b25b2cd7cde3d2f9914d0cbe94e Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Fri, 15 Nov 2024 10:24:17 +0200 Subject: [PATCH 1/6] Make get_scene_coverage() a private function --- trollflow2/plugins/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/trollflow2/plugins/__init__.py b/trollflow2/plugins/__init__.py index 8b3e031..8f89ab4 100644 --- a/trollflow2/plugins/__init__.py +++ b/trollflow2/plugins/__init__.py @@ -639,8 +639,8 @@ def _check_overall_coverage_for_area( Helper for covers(). """ area_path = "/product_list/areas/%s" % area - cov = get_scene_coverage(platform_name, start_time, end_time, - sensor, area) + cov = _get_scene_coverage(platform_name, start_time, end_time, + sensor, area) product_list['product_list']['areas'][area]['area_coverage_percent'] = cov if cov < min_coverage: logger.info( @@ -654,7 +654,7 @@ def _check_overall_coverage_for_area( f"{min_coverage:.2f}% - Carry on with {area:s}") -def get_scene_coverage(platform_name, start_time, end_time, sensor, area_id): +def _get_scene_coverage(platform_name, start_time, end_time, sensor, area_id): """Get scene area coverage in percentages.""" overpass = Pass(platform_name, start_time, end_time, instrument=sensor) area_def = get_area_def(area_id) @@ -1028,8 +1028,8 @@ def _product_meets_min_valid_data_fraction( end_time = prod.attrs["end_time"] sensor = prod.attrs["sensor"] if area_name not in exp_cov: - # get_scene_coverage uses %, convert to fraction - exp_cov[area_name] = get_scene_coverage( + # _get_scene_coverage uses %, convert to fraction + exp_cov[area_name] = _get_scene_coverage( platform_name, start_time, end_time, sensor, area_name)/100 exp_valid = exp_cov[area_name] if exp_valid == 0: From 6c8019c114eeb5dace3ead874304d07b34b4fd53 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Fri, 15 Nov 2024 10:25:17 +0200 Subject: [PATCH 2/6] Check trollsched in check_valid_data_fraction plugin --- trollflow2/plugins/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/trollflow2/plugins/__init__.py b/trollflow2/plugins/__init__.py index 8f89ab4..33f53f1 100644 --- a/trollflow2/plugins/__init__.py +++ b/trollflow2/plugins/__init__.py @@ -968,6 +968,11 @@ def check_valid_data_fraction(job): """ logger.info("Checking valid data fraction.") + if get_twilight_poly is None: + logger.error("Trollsched import failed, calculation of valid data fraction not possible") + logger.info("Keeping all products") + return + exp_cov = {} # As stated, this will trigger a computation. To prevent computing # multiple times, we should persist everything that needs to be persisted, From 6cb6aaed791a4c04b26fa20b94b541508c8c221b Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Fri, 15 Nov 2024 10:30:34 +0200 Subject: [PATCH 3/6] Update tests --- trollflow2/tests/test_trollflow2.py | 40 ++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/trollflow2/tests/test_trollflow2.py b/trollflow2/tests/test_trollflow2.py index 23cc8cf..7b9492a 100644 --- a/trollflow2/tests/test_trollflow2.py +++ b/trollflow2/tests/test_trollflow2.py @@ -1347,9 +1347,9 @@ def test_covers_complains_when_multiple_sensors_are_provided(self): """Test that the plugin complains when multiple sensors are provided.""" from trollflow2.plugins import covers - with mock.patch('trollflow2.plugins.get_scene_coverage') as get_scene_coverage, \ + with mock.patch('trollflow2.plugins._get_scene_coverage') as _get_scene_coverage, \ mock.patch('trollflow2.plugins.Pass'): - get_scene_coverage.return_value = 10.0 + _get_scene_coverage.return_value = 10.0 scn = _get_mocked_scene_with_properties() job = {"product_list": self.product_list, "input_mda": {"platform_name": "platform", @@ -1365,9 +1365,9 @@ def test_covers_does_not_complain_when_one_sensor_is_provided_as_a_sequence(self """Test that the plugin complains when multiple sensors are provided.""" from trollflow2.plugins import covers - with mock.patch('trollflow2.plugins.get_scene_coverage') as get_scene_coverage, \ + with mock.patch('trollflow2.plugins._get_scene_coverage') as _get_scene_coverage, \ mock.patch('trollflow2.plugins.Pass'): - get_scene_coverage.return_value = 10.0 + _get_scene_coverage.return_value = 10.0 scn = _get_mocked_scene_with_properties() job = {"product_list": self.product_list, "input_mda": {"platform_name": "platform", @@ -1383,19 +1383,19 @@ def test_metadata_is_read_from_scene(self): """Test that the scene and message metadata are merged correctly.""" from trollflow2.plugins import covers - with mock.patch('trollflow2.plugins.get_scene_coverage') as get_scene_coverage, \ + with mock.patch('trollflow2.plugins._get_scene_coverage') as _get_scene_coverage, \ mock.patch('trollflow2.plugins.Pass'): - get_scene_coverage.return_value = 10.0 + _get_scene_coverage.return_value = 10.0 scn = _get_mocked_scene_with_properties() job = {"product_list": self.product_list, "input_mda": {"platform_name": "platform"}, "scene": scn} covers(job) - get_scene_coverage.assert_called_with(job["input_mda"]["platform_name"], - scn.start_time, - scn.end_time, - list(scn.sensor_names)[0], - "omerc_bb") + _get_scene_coverage.assert_called_with(job["input_mda"]["platform_name"], + scn.start_time, + scn.end_time, + list(scn.sensor_names)[0], + "omerc_bb") def test_covers(self): """Test coverage.""" @@ -1429,14 +1429,14 @@ def test_covers_uses_only_one_sensor(self): "scene": scn} job2 = copy.deepcopy(job) - with mock.patch('trollflow2.plugins.get_scene_coverage') as get_scene_coverage, \ + with mock.patch('trollflow2.plugins._get_scene_coverage') as _get_scene_coverage, \ mock.patch('trollflow2.plugins.Pass'): - get_scene_coverage.return_value = 10.0 + _get_scene_coverage.return_value = 10.0 covers(job) - get_scene_coverage.assert_called_with(input_mda['platform_name'], - input_mda['start_time'], - input_mda['end_time'], - 'avhrr-4', 'omerc_bb') + _get_scene_coverage.assert_called_with(input_mda['platform_name'], + input_mda['start_time'], + input_mda['end_time'], + 'avhrr-4', 'omerc_bb') del job2["product_list"]["product_list"]["areas"]["euron1"]["min_coverage"] del job2["product_list"]["product_list"]["min_coverage"] @@ -1446,7 +1446,7 @@ def test_covers_uses_only_one_sensor(self): def test_scene_coverage(self): """Test scene coverage.""" - from trollflow2.plugins import get_scene_coverage + from trollflow2.plugins import _get_scene_coverage with mock.patch('trollflow2.plugins.get_area_def') as get_area_def, \ mock.patch('trollflow2.plugins.Pass') as ts_pass: area_coverage = mock.MagicMock() @@ -1455,7 +1455,7 @@ def test_scene_coverage(self): overpass.area_coverage = area_coverage ts_pass.return_value = overpass get_area_def.return_value = 6 - res = get_scene_coverage(1, 2, 3, 4, 5) + res = _get_scene_coverage(1, 2, 3, 4, 5) self.assertEqual(res, 100 * 0.2) ts_pass.assert_called_with(1, 2, 3, instrument=4) get_area_def.assert_called_with(5) @@ -2185,7 +2185,7 @@ def test_valid_filter(caplog, sc_3a_3b): job2 = copy.deepcopy(job) prods2 = job2['product_list']['product_list']['areas']['euron1']['products'] - with mock.patch("trollflow2.plugins.get_scene_coverage") as tpg, \ + with mock.patch("trollflow2.plugins._get_scene_coverage") as tpg, \ caplog.at_level(logging.DEBUG): tpg.return_value = 100 check_valid_data_fraction(job) From c6fc9e92100d5486f79634bd94c747dc9dc4240a Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Fri, 15 Nov 2024 11:16:09 +0200 Subject: [PATCH 4/6] Refactor test_valid_filter() --- trollflow2/tests/test_trollflow2.py | 67 +++++++++++++++++++---------- 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/trollflow2/tests/test_trollflow2.py b/trollflow2/tests/test_trollflow2.py index 7b9492a..0858f23 100644 --- a/trollflow2/tests/test_trollflow2.py +++ b/trollflow2/tests/test_trollflow2.py @@ -2168,22 +2168,11 @@ def sc_3a_3b(): return scene -def test_valid_filter(caplog, sc_3a_3b): - """Test filter for minimum fraction of valid data.""" - from trollflow2.launcher import yaml +def test_valid_filter_full_coverage(caplog, sc_3a_3b): + """Test filter for minimum fraction of valid data with full coverage.""" from trollflow2.plugins import check_valid_data_fraction - product_list = yaml.safe_load(yaml_test3) - job = {} - job['scene'] = sc_3a_3b - job['product_list'] = product_list.copy() - job['input_mda'] = input_mda.copy() - job['resampled_scenes'] = {"euron1": sc_3a_3b} - prods = job['product_list']['product_list']['areas']['euron1']['products'] - for p in ("NIR016", "IR037", "absent"): - prods[p] = {"min_valid_data_fraction": 40} - job2 = copy.deepcopy(job) - prods2 = job2['product_list']['product_list']['areas']['euron1']['products'] + job, prods = _create_valid_filter_job_and_prods(sc_3a_3b) with mock.patch("trollflow2.plugins._get_scene_coverage") as tpg, \ caplog.at_level(logging.DEBUG): @@ -2194,18 +2183,52 @@ def test_valid_filter(caplog, sc_3a_3b): assert "removing NIR016 for area euron1" in caplog.text assert "keeping IR037 for area euron1" in caplog.text assert "product absent not found, already removed" in caplog.text - tpg.reset_mock() + + +def test_valid_filter_small_coverage(caplog, sc_3a_3b): + """Test filter for minimum fraction of valid data with small coverage.""" + from trollflow2.plugins import check_valid_data_fraction + + job, prods = _create_valid_filter_job_and_prods(sc_3a_3b) + + with mock.patch("trollflow2.plugins._get_scene_coverage") as tpg, \ + caplog.at_level(logging.DEBUG): tpg.return_value = 1 - check_valid_data_fraction(job2) + check_valid_data_fraction(job) assert "inaccurate coverage estimate suspected!" in caplog.text - assert "NIR016" in prods2 - assert "IR037" in prods2 - tpg.reset_mock() + assert "NIR016" in prods + assert "IR037" in prods + + +def test_valid_filter_zero_coverage(caplog, sc_3a_3b): + """Test filter for minimum fraction of valid data without any coverage.""" + from trollflow2.plugins import check_valid_data_fraction + + job, prods = _create_valid_filter_job_and_prods(sc_3a_3b) + + with mock.patch("trollflow2.plugins._get_scene_coverage") as tpg, \ + caplog.at_level(logging.DEBUG): tpg.return_value = 0 - check_valid_data_fraction(job2) + check_valid_data_fraction(job) assert "no expected coverage at all, removing" in caplog.text - assert "NIR016" not in prods2 - assert "IR037" not in prods2 + assert "NIR016" not in prods + assert "IR037" not in prods + + +def _create_valid_filter_job_and_prods(sc_3a_3b): + from trollflow2.launcher import yaml + product_list = yaml.safe_load(yaml_test3) + job = {} + job['scene'] = sc_3a_3b + job['product_list'] = product_list.copy() + job['input_mda'] = input_mda.copy() + job['resampled_scenes'] = {"euron1": sc_3a_3b} + + prods = job['product_list']['product_list']['areas']['euron1']['products'] + for p in ("NIR016", "IR037", "absent"): + prods[p] = {"min_valid_data_fraction": 40} + + return job, prods def test_persisted(sc_3a_3b): From 07023a79121ae7d9a1f6e9156a38383b99cad84d Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Fri, 15 Nov 2024 11:22:21 +0200 Subject: [PATCH 5/6] Check that missing pytroll-schedule emits log message --- trollflow2/tests/test_trollflow2.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/trollflow2/tests/test_trollflow2.py b/trollflow2/tests/test_trollflow2.py index 0858f23..4522b51 100644 --- a/trollflow2/tests/test_trollflow2.py +++ b/trollflow2/tests/test_trollflow2.py @@ -2215,6 +2215,22 @@ def test_valid_filter_zero_coverage(caplog, sc_3a_3b): assert "IR037" not in prods +def test_valid_filter_no_trollsched(caplog, monkeypatch, sc_3a_3b): + """Test filter for minimum fraction of valid data with full coverage.""" + monkeypatch.setattr("trollsched.spherical.get_twilight_poly", None) + from trollflow2.plugins import check_valid_data_fraction + + job, prods = _create_valid_filter_job_and_prods(sc_3a_3b) + + with mock.patch("trollflow2.plugins._get_scene_coverage") as tpg, \ + caplog.at_level(logging.DEBUG): + tpg.return_value = 100 + check_valid_data_fraction(job) + + assert "Trollsched import failed" in caplog.text + assert "Keeping all products" in caplog.text + + def _create_valid_filter_job_and_prods(sc_3a_3b): from trollflow2.launcher import yaml product_list = yaml.safe_load(yaml_test3) From 4058756893421c247ad399895a6cf23cc9a7ecd6 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Mon, 18 Nov 2024 13:14:31 +0200 Subject: [PATCH 6/6] Add monkeypatching so that it works also for runnin all the tests --- trollflow2/tests/test_trollflow2.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/trollflow2/tests/test_trollflow2.py b/trollflow2/tests/test_trollflow2.py index 86928cd..7ae131b 100644 --- a/trollflow2/tests/test_trollflow2.py +++ b/trollflow2/tests/test_trollflow2.py @@ -2223,7 +2223,10 @@ def test_valid_filter_zero_coverage(caplog, sc_3a_3b): def test_valid_filter_no_trollsched(caplog, monkeypatch, sc_3a_3b): """Test filter for minimum fraction of valid data with full coverage.""" + # This is needed when only this test is run monkeypatch.setattr("trollsched.spherical.get_twilight_poly", None) + # ... and this when all the tests are run + monkeypatch.setattr("trollflow2.plugins.get_twilight_poly", None) from trollflow2.plugins import check_valid_data_fraction job, prods = _create_valid_filter_job_and_prods(sc_3a_3b)