Fix message for invalid input result (#312)

* Fix message for invalid input Signed-off-by: Tyler Gu <[email protected]> * Fix handling for invalid input Signed-off-by: Tyler Gu <[email protected]> * Fix test for operator log Signed-off-by: Tyler Gu <[email protected]> --------- Signed-off-by: Tyler Gu <[email protected]>
xlab-uiuc · Jan 23, 2024 · e48cae8 · e48cae8
1 parent 72df2ce
commit e48cae8
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 20 deletions.
diff --git a/acto/checker/impl/tests/test_operator_log.py b/acto/checker/impl/tests/test_operator_log.py
@@ -1,16 +1,19 @@
+from typing import Optional
+
 import pytest
 
 from acto.checker.impl.operator_log import OperatorLogChecker
 from acto.checker.impl.tests import load_snapshot
 from acto.common import PropertyPath
-from acto.result import InvalidInputResult, OracleResult
+from acto.result import InvalidInputResult
 from acto.snapshot import Snapshot
 
 # Crash checker is stateless, so we can use the same instance for all tests
 checker = OperatorLogChecker()
 
 
-def checker_func(s: Snapshot, prev_s: Snapshot) -> OracleResult:
+def checker_func(s: Snapshot, prev_s: Snapshot) -> Optional[InvalidInputResult]:
+    """Helper function to check the checker."""
     assert s.operator_log != []
     return checker.check(0, s, prev_s)
 
@@ -21,16 +24,13 @@ def checker_func(s: Snapshot, prev_s: Snapshot) -> OracleResult:
         enumerate(
             [
                 InvalidInputResult(
-                    message="Found no matching fields for input",
                     responsible_property=PropertyPath([]),
                 ),
                 None,
                 InvalidInputResult(
-                    message="Found no matching fields for input",
                     responsible_property=PropertyPath([]),
                 ),
                 InvalidInputResult(
-                    message="Found no matching fields for input",
                     responsible_property=PropertyPath(
                         [
                             "spec",
@@ -49,7 +49,8 @@ def checker_func(s: Snapshot, prev_s: Snapshot) -> OracleResult:
         )
     ),
 )
-def test_check(test_case_id, result_dict):
+def test_operator_log_checker(test_case_id, result_dict):
+    """Test operator log checker."""
     snapshot = load_snapshot("operator_log", test_case_id)
     snapshot_prev = load_snapshot("operator_log", test_case_id, load_prev=True)
     oracle_result = checker_func(snapshot, snapshot_prev)

diff --git a/acto/engine.py b/acto/engine.py
@@ -417,6 +417,7 @@ def run_trial(
         )
 
         generation = 0
+        trial_id = f"trial-{self.worker_id + self.sequence_base:02d}-{self.curr_trial:04d}"
         while (
             generation < num_mutation
         ):  # every iteration gets a new list of next tests
@@ -439,7 +440,7 @@ def run_trial(
                 # break and move to the next trial
                 if test_groups is None:
                     return TrialResult(
-                        trial_id=f"trial-{self.worker_id + self.sequence_base:02d}-{self.curr_trial:04d}",
+                        trial_id=trial_id,
                         duration=time.time() - trial_start_time,
                         error=None,
                     )
@@ -503,30 +504,35 @@ def run_trial(
                         ):
                             logger.error("Connection refused, exiting")
                             return TrialResult(
-                                trial_id=f"trial-{self.worker_id + self.sequence_base:02d}-{self.curr_trial:04d}",
+                                trial_id=trial_id,
                                 duration=time.time() - trial_start_time,
                                 error=None,
                             )
-                        if run_result.oracle_result.is_error():
+                        if (
+                            run_result.is_invalid_input()
+                            and run_result.oracle_result.health is None
+                            and run_result.oracle_result.crash is None
+                            and run_result.oracle_result.custom is None
+                        ):
+                            logger.info("Setup produced invalid input")
+                            self.snapshots.pop()
+                            group.discard_testcase(self.discarded_testcases)
+                            curr_input_with_schema = self.revert(
+                                runner, checker, generation
+                            )
+                            generation += 1
+                        elif run_result.oracle_result.is_error():
                             group.discard_testcase(self.discarded_testcases)
                             # before return, run the recovery test case
                             run_result.oracle_result.differential = self.run_recovery(  # pylint: disable=assigning-non-slot
                                 runner
                             )
                             generation += 1
                             return TrialResult(
-                                trial_id=f"trial-{self.worker_id + self.sequence_base:02d}-{self.curr_trial:04d}",
+                                trial_id=trial_id,
                                 duration=time.time() - trial_start_time,
                                 error=run_result.oracle_result,
                             )
-                        if run_result.is_invalid_input():
-                            logger.info("Setup produced invalid input")
-                            self.snapshots.pop()
-                            group.discard_testcase(self.discarded_testcases)
-                            curr_input_with_schema = self.revert(
-                                runner, checker, generation
-                            )
-                            generation += 1
                         elif run_result.cli_status == CliStatus.UNCHANGED:
                             logger.info("Setup produced unchanged input")
                             group.discard_testcase(self.discarded_testcases)
@@ -610,7 +616,12 @@ def run_testcases(
             logger.error("Connection refused, exiting")
             return run_result, generation
 
-        if run_result.is_invalid_input():
+        if (
+            run_result.is_invalid_input()
+            and run_result.oracle_result.health is None
+            and run_result.oracle_result.crash is None
+            and run_result.oracle_result.custom is None
+        ):
             # If the result indicates our input is invalid, we need to first run
             # revert to go back to previous system state
             logger.debug("Invalid input, revert")

diff --git a/acto/result.py b/acto/result.py
@@ -87,7 +87,7 @@ class InvalidInputResult(OracleResult):
 
     message: str = pydantic.Field(
         description="The message of the oracle run",
-        default="Found no matching fields for input",
+        default="Invalid input detected",
     )
     responsible_property: Optional[PropertyPath] = pydantic.Field(
         description="The fields that were not present in the input",