Check the status of providers if they provide a status check endpoint

projects/etos_suite_runner/src/etos_suite_runner/lib/executor.py

@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Executor handler module."""
+
 import logging
 import os
 from json import JSONDecodeError, dumps
@@ -23,6 +24,7 @@
 from etos_lib import ETOS
 from etos_lib.opentelemetry.semconv import Attributes as SemConvAttributes
 from opentelemetry import trace
+from requests import Response
 from requests.auth import HTTPBasicAuth, HTTPDigestAuth
 from requests.exceptions import ConnectionError as RequestsConnectionError
 from requests.exceptions import HTTPError
@@ -70,7 +72,10 @@ def __decrypt(self, password: Union[str, dict]) -> str:
         return Fernet(key).decrypt(password_value).decode()
 
     def __auth(
-        self, username: str, password: str, type: str = "basic"  # pylint:disable=redefined-builtin
+        self,
+        username: str,
+        password: str,
+        type: str = "basic",  # pylint:disable=redefined-builtin
     ) -> Union[HTTPBasicAuth, HTTPDigestAuth]:
         """Create an authentication for HTTP request.
 
@@ -84,25 +89,30 @@ def __auth(
             return HTTPBasicAuth(username, password)
         return HTTPDigestAuth(username, password)
 
+    def do_request(self, request: dict) -> Response:
+        """Make an HTTP request based on a request dictionary."""
+        if request.get("auth"):
+            request["auth"] = self.__auth(**request["auth"])
+        method = getattr(self.etos.http, request.pop("method").lower())
+        response = method(**request)
+        response.raise_for_status()
+        return response
+
     def run_tests(self, test_suite: dict) -> None:
         """Run tests in jenkins.
 
         :param test_suite: Tests to execute.
         """
         executor = test_suite.get("executor")
         request = executor.get("request")
-        if request.get("auth"):
-            request["auth"] = self.__auth(**request["auth"])
-        method = getattr(self.etos.http, request.pop("method").lower())
         span_name = "start_execution_space"
         with self.tracer.start_as_current_span(span_name, kind=trace.SpanKind.CLIENT) as span:
             span.set_attribute(
                 SemConvAttributes.EXECUTOR_ID, executor["id"] if "id" in executor else ""
             )
             span.set_attribute("http.request.body", dumps(request))
             try:
-                response = method(**request)
-                response.raise_for_status()
+                response = self.do_request(request)
             except HTTPError as http_error:
                 try:
                     exc = TestStartException(http_error.response.json())

projects/etos_suite_runner/src/etos_suite_runner/lib/suite.py

@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Test suite handler."""
+
 import os
 import json
 import logging
@@ -32,6 +33,7 @@
 from jsontas.jsontas import JsonTas
 import opentelemetry
 from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+from requests.exceptions import ConnectionError as RequestsConnectionError, HTTPError
 
 from .esr_parameters import ESRParameters
 from .exceptions import EnvironmentProviderException
@@ -121,6 +123,51 @@ def outcome(self) -> dict:
             return self.test_suite_finished.get("data", {}).get("testSuiteOutcome", {})
         return {}
 
+    def _environment_active(self) -> bool:
+        """Check the status of the test environment."""
+        status_requests = (
+            (
+                self.environment["iut"].get("provider_id", "iut_provider"),
+                self.environment["iut"].get("status_request"),
+            ),
+            (
+                self.environment["executor"].get("provider_id", "execution_space_provider"),
+                self.environment["executor"].get("status_request"),
+            ),
+            (
+                self.environment["log_area"].get("provider_id", "log_area_provider"),
+                self.environment["log_area"].get("status_request"),
+            ),
+        )
+        executor = Executor(self.etos)
+        for name, request in status_requests:
+            if request is None:
+                continue
+            try:
+                response = executor.do_request(request)
+                response.json()
+            except RequestsConnectionError:
+                self.logger.exception(
+                    "Could not communicate with the status check endpoint for %r (%r)",
+                    name,
+                    request.get("url"),
+                )
+                return False
+            except HTTPError as http_error:
+                self.logger.error(
+                    "Got error code %r when communicating with %r (%r)",
+                    http_error.response.status_code,
+                    name,
+                    request.get("url"),
+                )
+                return False
+            except json.JSONDecodeError:
+                self.logger.exception(
+                    "Failed to parse the response from for %r (%r)", name, request.get("url")
+                )
+                return False
+        return True
+
     def _start(self, identifier: str) -> None:
         """Start ETR for this sub suite.
 
@@ -156,6 +203,12 @@ def _start(self, identifier: str) -> None:
                     if self.finished:
                         self.logger.info("ETOS test runner has finished", extra={"user_log": True})
                         break
+                    if not self._environment_active():
+                        self.logger.error(
+                            "ETOS test environment is no longer active! Canceling.",
+                            extra={"user_log": True},
+                        )
+                        break
             finally:
                 self.release(identifier)