Merge pull request #732 from glatterf42/enh/ci/flaky-tests

Handling flaky tests

message_ix/tests/test_feature_bound_activity_shares.py

@@ -1,9 +1,21 @@
+import os
+import platform
+
 import numpy as np
 import pandas as pd
+import pytest
 
 from message_ix import Scenario
 from message_ix.testing import SCENARIO
 
+FLAKY = pytest.mark.flaky(
+    reruns=5,
+    rerun_delay=2,
+    condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Darwin",
+    reason="Flaky; see iiasa/message_ix#731",
+)
+
+
 # First model year of the Dantzig scenario
 _year = 1963
 
@@ -45,6 +57,7 @@ def test_add_bound_activity_up(message_test_mp):
     assert new_obj >= orig_obj
 
 
+@FLAKY
 def test_add_bound_activity_up_all_modes(message_test_mp):
     # This test specifically has two solutions for which the `OBJ` function is the same
     # therefore the lpmethod must be set to "2".
@@ -228,7 +241,8 @@ def add_data(s, map_df):
     assert new_obj >= orig_obj
 
 
-def test_share_commodity_lo(message_test_mp):
+@FLAKY
+def test_commodity_share_lo(message_test_mp):
     scen = Scenario(message_test_mp, **SCENARIO["dantzig"]).clone()
     scen.solve(quiet=True)
 
@@ -344,6 +358,7 @@ def calc_share(s):
     assert new_obj >= orig_obj
 
 
+@FLAKY
 def test_add_share_mode_lo(message_test_mp):
     scen = Scenario(message_test_mp, **SCENARIO["dantzig"]).clone()
     scen.solve(quiet=True)

message_ix/tests/test_feature_price_emission.py

@@ -1,7 +1,19 @@
+import os
+import platform
+
 import numpy.testing as npt
+import pytest
 
 from message_ix import Scenario
 
+FLAKY = pytest.mark.flaky(
+    reruns=5,
+    rerun_delay=2,
+    condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Darwin",
+    reason="Flaky; see iiasa/message_ix#731",
+)
+
+
 MODEL = "test_emissions_price"
 
 
@@ -74,6 +86,7 @@ def test_no_constraint(test_mp):
     assert scen.var("PRICE_EMISSION").empty
 
 
+@FLAKY
 def test_cumulative_equidistant(test_mp):
     scen = Scenario(test_mp, MODEL, "cum_equidistant", version="new")
     years = [2020, 2030, 2040]
@@ -110,6 +123,7 @@ def test_per_period_equidistant(test_mp):
     npt.assert_allclose(scen.var("PRICE_EMISSION")["lvl"], [1] * 3)
 
 
+@FLAKY
 def test_cumulative_variable_periodlength(test_mp):
     scen = Scenario(test_mp, MODEL, "cum_equidistant", version="new")
     years = [2020, 2025, 2030, 2040]
@@ -128,6 +142,7 @@ def test_cumulative_variable_periodlength(test_mp):
     npt.assert_allclose(obs, [1.05 ** (y - years[0]) for y in years])
 
 
+@FLAKY
 def test_per_period_variable_periodlength(test_mp):
     scen = Scenario(test_mp, MODEL, "cum_equidistant", version="new")
     years = [2020, 2025, 2030, 2040]
@@ -146,6 +161,7 @@ def test_per_period_variable_periodlength(test_mp):
     npt.assert_allclose(scen.var("PRICE_EMISSION")["lvl"].values, [1] * 4)
 
 
+@FLAKY
 def test_custom_type_variable_periodlength(test_mp):
     scen = Scenario(test_mp, MODEL, "cum_equidistant", version="new")
     years = [2020, 2025, 2030, 2040, 2050]

message_ix/tests/test_integration.py

@@ -1,3 +1,6 @@
+import os
+import platform
+
 import numpy as np
 import pytest
 from ixmp import Platform
@@ -84,6 +87,12 @@ def assert_multi_db(mp1, mp2):
     assert_frame_equal(scenario_list(mp1), scenario_list(mp2))
 
 
+@pytest.mark.flaky(
+    reruns=5,
+    rerun_delay=2,
+    condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Darwin",
+    reason="Flaky; see iiasa/message_ix#731",
+)
 def test_multi_db_run(tmpdir):
     # create a new instance of the transport problem and solve it
     mp1 = Platform(driver="hsqldb", path=tmpdir / "mp1")

message_ix/tests/test_legacy_version.py

@@ -1,10 +1,20 @@
+import os
+import platform
+
 import numpy as np
+import pytest
 from ixmp import Platform
 from ixmp.testing import create_test_platform
 
 from message_ix import Scenario
 
 
+@pytest.mark.flaky(
+    reruns=5,
+    rerun_delay=2,
+    condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Darwin",
+    reason="Flaky; see iiasa/message_ix#731",
+)
 def test_solve_legacy_scenario(tmp_path, test_data_path):
     db_path = create_test_platform(tmp_path, test_data_path, "legacy")
     mp = Platform(backend="jdbc", driver="hsqldb", path=db_path)

message_ix/tests/test_macro.py

@@ -1,3 +1,5 @@
+import os
+import platform
 from pathlib import Path
 
 import numpy as np
@@ -10,6 +12,13 @@
 from message_ix.reporting import Quantity
 from message_ix.testing import SCENARIO, make_westeros
 
+FLAKY = pytest.mark.flaky(
+    reruns=5,
+    rerun_delay=2,
+    condition="GITHUB_ACTIONS" in os.environ and platform.system() == "Darwin",
+    reason="Flaky; see iiasa/message_ix#731",
+)
+
 # Fixtures
 
 
@@ -51,6 +60,7 @@ def westeros_not_solved(_ws):
 # Tests
 
 
+@FLAKY
 def test_calc_valid_data_file(westeros_solved, w_data_path):
     c = macro.prepare_computer(westeros_solved, data=w_data_path)
     c.get("check all")
@@ -97,6 +107,7 @@ def test_calc_no_solution(westeros_not_solved, w_data_path):
         macro.prepare_computer(s, data=w_data_path)
 
 
+@FLAKY
 def test_config(westeros_solved, w_data_path):
     c = macro.prepare_computer(westeros_solved, data=w_data_path)
     assert "config::macro" in c.graph
@@ -172,6 +183,7 @@ def test_calc_data_missing_datapoint(westeros_solved, w_data):
 #
 
 
+@FLAKY
 @pytest.mark.parametrize(
     "key, test, expected",
     (
@@ -236,6 +248,7 @@ def test_init(message_test_mp):
     assert "COST_ACCOUNTING_NODAL" in scen.equ_list()
 
 
+@FLAKY
 def test_add_model_data(westeros_solved, w_data_path):
     base = westeros_solved
     clone = base.clone("foo", "bar", keep_solution=False)

message_ix/tests/test_tutorials.py

@@ -30,7 +30,7 @@
 
 # Affects all tests in the file
 pytestmark = pytest.mark.flaky(
-    reruns=3,
+    reruns=5,
     rerun_delay=2,
     condition=GHA,
     reason="Flaky; fails occasionally on GitHub Actions runners",
@@ -121,7 +121,7 @@ def nb_path(request, tutorial_path):
 
 def default_args():
     """Default arguments for :func:`.run_notebook."""
-    if GHA and sys.platform in ("darwin", "win32"):
+    if GHA:
         # Use a longer timeout
         return dict(timeout=30)
     else: