Merge pull request #40 from enix/fix_keep_servers_after_fail

Fix --keep-servers-after-fail option by retrieving test result inside fixture

Author: Zempashi

tests/conftest.py

@@ -14,16 +14,18 @@
     install_galaxy_deps,
     run_ansible_playbook,
 )
-from tests.helpers.provider import cluster, provider  # noqa: F401, Those are fixtures
 from tests.helpers.terraform import TerraformCompose
 from tests.helpers.vagrant import LocalVagrant
 
 DEFAULT_OS = ["ubuntu2204"]
 ALL_OS = ["ubuntu2004", "ubuntu2204", "debian11"]
 
 
+pytest_plugins = ["tests.helpers.provider"]
+
+
 def pytest_addoption(parser):
-    TRUE_VALUES = ["true", "yes", "y", "1"]
+    TRUE_VALUES = ["true", "yes", "y", "1", True]
     parser.addoption(
         "--keep-servers",
         dest="keep_servers",
@@ -36,7 +38,7 @@ def pytest_addoption(parser):
         "--keep-servers-after-fail",
         dest="keep_servers_after_fail",
         nargs="?",
-        type=bool,
+        type=lambda arg: arg in TRUE_VALUES,
         const=True,
         default=os.environ.get("KEEP_SERVERS_AFTER_FAIL", "true").lower()
         in TRUE_VALUES,

tests/helpers/provider.py

@@ -1,7 +1,23 @@
 import os
+from typing import Dict
 
 import pytest
 import tenacity
+from pytest import CollectReport, StashKey
+
+# https://docs.pytest.org/en/latest/example/simple.html#making-test-result-information-available-in-fixtures
+phase_report_key = StashKey[Dict[str, CollectReport]]()
+
+
+@pytest.hookimpl(tryfirst=True, hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+    # execute all other hooks to obtain the report object
+    outcome = yield
+    rep = outcome.get_result()
+
+    # store test results for each phase of a call, which can
+    # be "setup", "call", "teardown"
+    item.stash.setdefault(phase_report_key, {})[rep.when] = rep
 
 
 @pytest.fixture
@@ -15,18 +31,26 @@ def provider(request):
 
 
 @tenacity.retry(reraise=True, stop=tenacity.stop_after_attempt(2))
-def cluster_spawn(provider):
+def cluster_spawn(provider, keep_servers):
     provider.init()
+    if not keep_servers:
+        provider.destroy()
     provider.apply()
 
 
 @pytest.fixture
 def cluster(request, provider, operating_system):
+    keep_servers = request.config.getoption("keep_servers")
+    keep_after_fail = request.config.getoption("keep_servers_after_fail")
     try:
-        cluster_spawn(provider)
+        cluster_spawn(provider, keep_servers)
         yield provider.cluster()
-        if not request.config.getoption("keep_servers"):
+        report = request.node.stash[phase_report_key]
+        if "call" in report and report["call"].failed:
+            if not keep_after_fail:
+                provider.destroy()
+        elif not keep_servers:
             provider.destroy()
     except Exception:
-        if not request.config.getoption("keep_servers_after_fail"):
+        if not keep_after_fail:
             provider.destroy()