diff --git a/acto/checker/impl/tests/test_operator_log.py b/acto/checker/impl/tests/test_operator_log.py index 1967702242..6144b2e374 100644 --- a/acto/checker/impl/tests/test_operator_log.py +++ b/acto/checker/impl/tests/test_operator_log.py @@ -1,16 +1,19 @@ +from typing import Optional + import pytest from acto.checker.impl.operator_log import OperatorLogChecker from acto.checker.impl.tests import load_snapshot from acto.common import PropertyPath -from acto.result import InvalidInputResult, OracleResult +from acto.result import InvalidInputResult from acto.snapshot import Snapshot # Crash checker is stateless, so we can use the same instance for all tests checker = OperatorLogChecker() -def checker_func(s: Snapshot, prev_s: Snapshot) -> OracleResult: +def checker_func(s: Snapshot, prev_s: Snapshot) -> Optional[InvalidInputResult]: + """Helper function to check the checker.""" assert s.operator_log != [] return checker.check(0, s, prev_s) @@ -21,16 +24,13 @@ def checker_func(s: Snapshot, prev_s: Snapshot) -> OracleResult: enumerate( [ InvalidInputResult( - message="Found no matching fields for input", responsible_property=PropertyPath([]), ), None, InvalidInputResult( - message="Found no matching fields for input", responsible_property=PropertyPath([]), ), InvalidInputResult( - message="Found no matching fields for input", responsible_property=PropertyPath( [ "spec", @@ -49,7 +49,8 @@ def checker_func(s: Snapshot, prev_s: Snapshot) -> OracleResult: ) ), ) -def test_check(test_case_id, result_dict): +def test_operator_log_checker(test_case_id, result_dict): + """Test operator log checker.""" snapshot = load_snapshot("operator_log", test_case_id) snapshot_prev = load_snapshot("operator_log", test_case_id, load_prev=True) oracle_result = checker_func(snapshot, snapshot_prev) diff --git a/acto/engine.py b/acto/engine.py index 2c5db4ae23..61205f7a9d 100644 --- a/acto/engine.py +++ b/acto/engine.py @@ -417,6 +417,7 @@ def run_trial( ) generation = 0 + trial_id = f"trial-{self.worker_id + self.sequence_base:02d}-{self.curr_trial:04d}" while ( generation < num_mutation ): # every iteration gets a new list of next tests @@ -439,7 +440,7 @@ def run_trial( # break and move to the next trial if test_groups is None: return TrialResult( - trial_id=f"trial-{self.worker_id + self.sequence_base:02d}-{self.curr_trial:04d}", + trial_id=trial_id, duration=time.time() - trial_start_time, error=None, ) @@ -503,11 +504,24 @@ def run_trial( ): logger.error("Connection refused, exiting") return TrialResult( - trial_id=f"trial-{self.worker_id + self.sequence_base:02d}-{self.curr_trial:04d}", + trial_id=trial_id, duration=time.time() - trial_start_time, error=None, ) - if run_result.oracle_result.is_error(): + if ( + run_result.is_invalid_input() + and run_result.oracle_result.health is None + and run_result.oracle_result.crash is None + and run_result.oracle_result.custom is None + ): + logger.info("Setup produced invalid input") + self.snapshots.pop() + group.discard_testcase(self.discarded_testcases) + curr_input_with_schema = self.revert( + runner, checker, generation + ) + generation += 1 + elif run_result.oracle_result.is_error(): group.discard_testcase(self.discarded_testcases) # before return, run the recovery test case run_result.oracle_result.differential = self.run_recovery( # pylint: disable=assigning-non-slot @@ -515,18 +529,10 @@ def run_trial( ) generation += 1 return TrialResult( - trial_id=f"trial-{self.worker_id + self.sequence_base:02d}-{self.curr_trial:04d}", + trial_id=trial_id, duration=time.time() - trial_start_time, error=run_result.oracle_result, ) - if run_result.is_invalid_input(): - logger.info("Setup produced invalid input") - self.snapshots.pop() - group.discard_testcase(self.discarded_testcases) - curr_input_with_schema = self.revert( - runner, checker, generation - ) - generation += 1 elif run_result.cli_status == CliStatus.UNCHANGED: logger.info("Setup produced unchanged input") group.discard_testcase(self.discarded_testcases) @@ -610,7 +616,12 @@ def run_testcases( logger.error("Connection refused, exiting") return run_result, generation - if run_result.is_invalid_input(): + if ( + run_result.is_invalid_input() + and run_result.oracle_result.health is None + and run_result.oracle_result.crash is None + and run_result.oracle_result.custom is None + ): # If the result indicates our input is invalid, we need to first run # revert to go back to previous system state logger.debug("Invalid input, revert") diff --git a/acto/result.py b/acto/result.py index c9aaface50..dc5f3d79e4 100644 --- a/acto/result.py +++ b/acto/result.py @@ -87,7 +87,7 @@ class InvalidInputResult(OracleResult): message: str = pydantic.Field( description="The message of the oracle run", - default="Found no matching fields for input", + default="Invalid input detected", ) responsible_property: Optional[PropertyPath] = pydantic.Field( description="The fields that were not present in the input",