From 9e5845a8b94e80947c661ecedda04776e6327b53 Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Wed, 2 Aug 2023 15:54:25 +0200 Subject: [PATCH 1/2] Increase number of reruns and timeout time for ubuntu runners --- message_ix/tests/test_tutorials.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/message_ix/tests/test_tutorials.py b/message_ix/tests/test_tutorials.py index 1ade463cd..29ebb0fb9 100644 --- a/message_ix/tests/test_tutorials.py +++ b/message_ix/tests/test_tutorials.py @@ -30,7 +30,7 @@ # Affects all tests in the file pytestmark = pytest.mark.flaky( - reruns=3, + reruns=5, rerun_delay=2, condition=GHA, reason="Flaky; fails occasionally on GitHub Actions runners", @@ -121,7 +121,7 @@ def nb_path(request, tutorial_path): def default_args(): """Default arguments for :func:`.run_notebook.""" - if GHA and sys.platform in ("darwin", "win32"): + if GHA: # Use a longer timeout return dict(timeout=30) else: From d65e36bc9bbe0d42d68408056bbf8c2b271e7c0a Mon Sep 17 00:00:00 2001 From: Fridolin Glatter Date: Fri, 4 Aug 2023 09:59:57 +0200 Subject: [PATCH 2/2] Mark tests as flaky as proposed in #731 --- .../tests/test_feature_bound_activity_shares.py | 17 ++++++++++++++++- message_ix/tests/test_feature_price_emission.py | 16 ++++++++++++++++ message_ix/tests/test_integration.py | 9 +++++++++ message_ix/tests/test_legacy_version.py | 10 ++++++++++ message_ix/tests/test_macro.py | 13 +++++++++++++ 5 files changed, 64 insertions(+), 1 deletion(-) diff --git a/message_ix/tests/test_feature_bound_activity_shares.py b/message_ix/tests/test_feature_bound_activity_shares.py index 089e34cf9..fce014d1f 100644 --- a/message_ix/tests/test_feature_bound_activity_shares.py +++ b/message_ix/tests/test_feature_bound_activity_shares.py @@ -1,9 +1,21 @@ +import os +import platform + import numpy as np import pandas as pd +import pytest from message_ix import Scenario from message_ix.testing import SCENARIO +FLAKY = pytest.mark.flaky( + reruns=5, + rerun_delay=2, + condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Darwin", + reason="Flaky; see iiasa/message_ix#731", +) + + # First model year of the Dantzig scenario _year = 1963 @@ -45,6 +57,7 @@ def test_add_bound_activity_up(message_test_mp): assert new_obj >= orig_obj +@FLAKY def test_add_bound_activity_up_all_modes(message_test_mp): # This test specifically has two solutions for which the `OBJ` function is the same # therefore the lpmethod must be set to "2". @@ -228,7 +241,8 @@ def add_data(s, map_df): assert new_obj >= orig_obj -def test_share_commodity_lo(message_test_mp): +@FLAKY +def test_commodity_share_lo(message_test_mp): scen = Scenario(message_test_mp, **SCENARIO["dantzig"]).clone() scen.solve(quiet=True) @@ -344,6 +358,7 @@ def calc_share(s): assert new_obj >= orig_obj +@FLAKY def test_add_share_mode_lo(message_test_mp): scen = Scenario(message_test_mp, **SCENARIO["dantzig"]).clone() scen.solve(quiet=True) diff --git a/message_ix/tests/test_feature_price_emission.py b/message_ix/tests/test_feature_price_emission.py index 718406ccc..0e12da892 100644 --- a/message_ix/tests/test_feature_price_emission.py +++ b/message_ix/tests/test_feature_price_emission.py @@ -1,7 +1,19 @@ +import os +import platform + import numpy.testing as npt +import pytest from message_ix import Scenario +FLAKY = pytest.mark.flaky( + reruns=5, + rerun_delay=2, + condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Darwin", + reason="Flaky; see iiasa/message_ix#731", +) + + MODEL = "test_emissions_price" @@ -74,6 +86,7 @@ def test_no_constraint(test_mp): assert scen.var("PRICE_EMISSION").empty +@FLAKY def test_cumulative_equidistant(test_mp): scen = Scenario(test_mp, MODEL, "cum_equidistant", version="new") years = [2020, 2030, 2040] @@ -110,6 +123,7 @@ def test_per_period_equidistant(test_mp): npt.assert_allclose(scen.var("PRICE_EMISSION")["lvl"], [1] * 3) +@FLAKY def test_cumulative_variable_periodlength(test_mp): scen = Scenario(test_mp, MODEL, "cum_equidistant", version="new") years = [2020, 2025, 2030, 2040] @@ -128,6 +142,7 @@ def test_cumulative_variable_periodlength(test_mp): npt.assert_allclose(obs, [1.05 ** (y - years[0]) for y in years]) +@FLAKY def test_per_period_variable_periodlength(test_mp): scen = Scenario(test_mp, MODEL, "cum_equidistant", version="new") years = [2020, 2025, 2030, 2040] @@ -146,6 +161,7 @@ def test_per_period_variable_periodlength(test_mp): npt.assert_allclose(scen.var("PRICE_EMISSION")["lvl"].values, [1] * 4) +@FLAKY def test_custom_type_variable_periodlength(test_mp): scen = Scenario(test_mp, MODEL, "cum_equidistant", version="new") years = [2020, 2025, 2030, 2040, 2050] diff --git a/message_ix/tests/test_integration.py b/message_ix/tests/test_integration.py index aeba8c01a..631ae73cf 100644 --- a/message_ix/tests/test_integration.py +++ b/message_ix/tests/test_integration.py @@ -1,3 +1,6 @@ +import os +import platform + import numpy as np import pytest from ixmp import Platform @@ -84,6 +87,12 @@ def assert_multi_db(mp1, mp2): assert_frame_equal(scenario_list(mp1), scenario_list(mp2)) +@pytest.mark.flaky( + reruns=5, + rerun_delay=2, + condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Darwin", + reason="Flaky; see iiasa/message_ix#731", +) def test_multi_db_run(tmpdir): # create a new instance of the transport problem and solve it mp1 = Platform(driver="hsqldb", path=tmpdir / "mp1") diff --git a/message_ix/tests/test_legacy_version.py b/message_ix/tests/test_legacy_version.py index 75819a8a6..d8b86bd2d 100644 --- a/message_ix/tests/test_legacy_version.py +++ b/message_ix/tests/test_legacy_version.py @@ -1,10 +1,20 @@ +import os +import platform + import numpy as np +import pytest from ixmp import Platform from ixmp.testing import create_test_platform from message_ix import Scenario +@pytest.mark.flaky( + reruns=5, + rerun_delay=2, + condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Darwin", + reason="Flaky; see iiasa/message_ix#731", +) def test_solve_legacy_scenario(tmp_path, test_data_path): db_path = create_test_platform(tmp_path, test_data_path, "legacy") mp = Platform(backend="jdbc", driver="hsqldb", path=db_path) diff --git a/message_ix/tests/test_macro.py b/message_ix/tests/test_macro.py index e19f56292..d0a2682c2 100644 --- a/message_ix/tests/test_macro.py +++ b/message_ix/tests/test_macro.py @@ -1,3 +1,5 @@ +import os +import platform from pathlib import Path import numpy as np @@ -10,6 +12,13 @@ from message_ix.reporting import Quantity from message_ix.testing import SCENARIO, make_westeros +FLAKY = pytest.mark.flaky( + reruns=5, + rerun_delay=2, + condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Darwin", + reason="Flaky; see iiasa/message_ix#731", +) + # Fixtures @@ -51,6 +60,7 @@ def westeros_not_solved(_ws): # Tests +@FLAKY def test_calc_valid_data_file(westeros_solved, w_data_path): c = macro.prepare_computer(westeros_solved, data=w_data_path) c.get("check all") @@ -97,6 +107,7 @@ def test_calc_no_solution(westeros_not_solved, w_data_path): macro.prepare_computer(s, data=w_data_path) +@FLAKY def test_config(westeros_solved, w_data_path): c = macro.prepare_computer(westeros_solved, data=w_data_path) assert "config::macro" in c.graph @@ -172,6 +183,7 @@ def test_calc_data_missing_datapoint(westeros_solved, w_data): # +@FLAKY @pytest.mark.parametrize( "key, test, expected", ( @@ -236,6 +248,7 @@ def test_init(message_test_mp): assert "COST_ACCOUNTING_NODAL" in scen.equ_list() +@FLAKY def test_add_model_data(westeros_solved, w_data_path): base = westeros_solved clone = base.clone("foo", "bar", keep_solution=False)