Deprecate should_error, renamed to expected_error and add success_ret…

…urncode for exitcode tests

docs/markdown/Unit-tests.md

@@ -132,7 +132,12 @@ In addition, sometimes a test fails set up so that it should fail even
 if it is marked as an expected failure. The GNU standard approach in
 this case is to exit the program with error code 99. Again, Meson will
 detect this and report these tests as `ERROR`, ignoring the setting of
-`should_fail`. This behavior was added in version 0.50.0.
+`should_fail`. This behavior was added in version 0.50.0. In version
+1.8.0 `should_fail` has been deprecated and renamed to `expected_fail`.
+
+In version 1.8.0, `success_returncode` has been introduced. This makes
+it possible to positively test for non-zero return codes. An example
+of this would be to test if failure injection is detected in a test.
 
 ## Testing tool
 

docs/markdown/snippets/expected_error.md

@@ -0,0 +1,3 @@
+## Deprecate `should_fail` and rename it to `expected_error`, also introduce `success_returncode`
+
+In 1.8.0 `should_fail` has been renamed to `expected_error`. Before 1.8.0, there was no way to positively test a command/binary returning error/non-zero return code, `success_returncode` has been introduced to achieve this.

docs/yaml/functions/benchmark.yaml

@@ -40,11 +40,29 @@ kwargs:
 
   should_fail:
     type: bool
+    deprecated: 1.8.0
     default: false
     description: |
       when true the test is considered passed if the
       executable returns a non-zero return value (i.e. reports an error)
 
+  expected_fail:
+    type: bool
+    since: 1.8.0
+    default: false
+    description: |
+      when true the test is considered passed if the
+      executable returns a non-zero return value (i.e. reports an error)
+
+  success_returncode:
+    type: int
+    since: 1.8.0
+    default: 0
+    description: |
+      the test is considered passed if the
+      executable returns the specified returncode,
+      only has effect when protocol is set to exitcode
+
   suite:
     type: str | list[str]
     description: |

mesonbuild/backend/backends.py

@@ -199,7 +199,8 @@ class TestSerialisation:
     is_parallel: bool
     cmd_args: T.List[str]
     env: mesonlib.EnvironmentVariables
-    should_fail: bool
+    expected_fail: bool
+    success_returncode: T.Optional[int]
     timeout: T.Optional[int]
     workdir: T.Optional[str]
     extra_paths: T.List[str]
@@ -1301,7 +1302,7 @@ def create_test_serialisation(self, tests: T.List['Test']) -> T.List[TestSeriali
             ts = TestSerialisation(t.get_name(), t.project_name, t.suite, cmd, is_cross,
                                    exe_wrapper, self.environment.need_exe_wrapper(),
                                    t.is_parallel, cmd_args, t_env,
-                                   t.should_fail, t.timeout, t.workdir,
+                                   t.expected_fail, t.success_returncode, t.timeout, t.workdir,
                                    extra_paths, t.protocol, t.priority,
                                    isinstance(exe, (build.Target, build.CustomTargetIndex)),
                                    isinstance(exe, build.Executable),

mesonbuild/interpreter/interpreter.py

@@ -2286,6 +2286,14 @@ def make_test(self, node: mparser.BaseNode,
         if kwargs['timeout'] <= 0:
             FeatureNew.single_use('test() timeout <= 0', '0.57.0', self.subproject, location=node)
 
+        expected_fail = False
+        if kwargs['should_fail'] is not None and kwargs['expected_fail'] is not None:
+            raise InvalidArguments('Tried to use both \'should_fail\' and \'expected_fail\'')
+        elif kwargs['should_fail'] is not None:
+            expected_fail = kwargs['should_fail']
+        elif kwargs['expected_fail'] is not None:
+            expected_fail = kwargs['expected_fail']
+
         prj = self.subproject if self.is_subproject() else self.build.project_name
 
         suite: T.List[str] = []
@@ -2302,7 +2310,8 @@ def make_test(self, node: mparser.BaseNode,
                     kwargs.get('is_parallel', False),
                     kwargs['args'],
                     env,
-                    kwargs['should_fail'],
+                    expected_fail,
+                    kwargs['success_returncode'],
                     kwargs['timeout'],
                     kwargs['workdir'],
                     kwargs['protocol'],

mesonbuild/interpreter/interpreterobjects.py

@@ -785,7 +785,7 @@ def __init__(self, name: str, project: str, suite: T.List[str],
                  is_parallel: bool,
                  cmd_args: T.List[T.Union[str, mesonlib.File, build.Target, ExternalProgram]],
                  env: mesonlib.EnvironmentVariables,
-                 should_fail: bool, timeout: int, workdir: T.Optional[str], protocol: str,
+                 expected_fail: bool, success_returncode: int, timeout: int, workdir: T.Optional[str], protocol: str,
                  priority: int, verbose: bool):
         super().__init__()
         self.name = name
@@ -796,7 +796,8 @@ def __init__(self, name: str, project: str, suite: T.List[str],
         self.is_parallel = is_parallel
         self.cmd_args = cmd_args
         self.env = env
-        self.should_fail = should_fail
+        self.expected_fail = expected_fail
+        self.success_returncode = success_returncode
         self.timeout = timeout
         self.workdir = workdir
         self.protocol = TestProtocol.from_str(protocol)

mesonbuild/interpreter/kwargs.py

@@ -39,7 +39,9 @@ class BaseTest(TypedDict):
     """Shared base for the Rust module."""
 
     args: T.List[T.Union[str, File, build.Target, ExternalProgram]]
-    should_fail: bool
+    should_fail: T.Optional[bool]
+    expected_fail: T.Optional[bool]
+    success_returncode: T.Optional[int]
     timeout: int
     workdir: T.Optional[str]
     depends: T.List[T.Union[build.CustomTarget, build.BuildTarget]]

mesonbuild/interpreter/type_checking.py

@@ -487,7 +487,9 @@ def link_whole_validator(values: T.List[T.Union[StaticLibrary, CustomTarget, Cus
 TEST_KWS: T.List[KwargInfo] = [
     KwargInfo('args', ContainerTypeInfo(list, (str, File, BuildTarget, CustomTarget, CustomTargetIndex, ExternalProgram)),
               listify=True, default=[]),
-    KwargInfo('should_fail', bool, default=False),
+    KwargInfo('should_fail', (bool, NoneType), deprecated='1.8.0', deprecated_message='Use expected_fail instead of should_fail'),
+    KwargInfo('expected_fail', (bool, NoneType), since='1.8.0'),
+    KwargInfo('success_returncode', (int, NoneType), since='1.8.0'),
     KwargInfo('timeout', int, default=30),
     KwargInfo('workdir', (str, NoneType), default=None,
               validator=lambda x: 'must be an absolute path' if not os.path.isabs(x) else None),

mesonbuild/mtest.py

@@ -940,7 +940,8 @@ def __init__(self, test: TestSerialisation, test_env: T.Dict[str, str],
         self.additional_error = ''
         self.cmd: T.Optional[T.List[str]] = None
         self.env = test_env
-        self.should_fail = test.should_fail
+        self.expected_fail = test.expected_fail
+        self.success_returncode = test.success_returncode
         self.project = test.project_name
         self.junit: T.Optional[et.ElementTree] = None
         self.is_parallel = is_parallel
@@ -988,7 +989,7 @@ def _complete(self) -> None:
         if self.res == TestResult.RUNNING:
             self.res = TestResult.OK
         assert isinstance(self.res, TestResult)
-        if self.should_fail and self.res in (TestResult.OK, TestResult.FAIL):
+        if self.expected_fail and self.res in (TestResult.OK, TestResult.FAIL):
             self.res = TestResult.UNEXPECTEDPASS if self.res is TestResult.OK else TestResult.EXPECTEDFAIL
         if self.stdo and not self.stdo.endswith('\n'):
             self.stdo += '\n'
@@ -1044,6 +1045,11 @@ class TestRunExitCode(TestRun):
     def complete(self) -> None:
         if self.res != TestResult.RUNNING:
             pass
+        elif self.success_returncode is not None:
+            if self.returncode == self.success_returncode:
+                self.res = TestResult.OK
+            else:
+                self.res = TestResult.FAIL
         elif self.returncode == GNU_SKIP_RETURNCODE:
             self.res = TestResult.SKIP
         elif self.returncode == GNU_ERROR_RETURNCODE:

test cases/common/278 success returncode/failing.c

@@ -0,0 +1,3 @@
+int main(void) {
+    return 1;
+}

test cases/common/278 success returncode/meson.build

@@ -0,0 +1,4 @@
+project('success returncode', 'c')
+
+exe = executable('prog', 'failing.c')
+test('errorcode', exe, success_returncode: 1)

test cases/common/68 should fail/meson.build

@@ -1,4 +1,5 @@
 project('should fail', 'c')
 
 exe = executable('prog', 'failing.c')
-test('failing', exe, should_fail : true)
+test('should-failing', exe, should_fail : true)
+test('expected-failing', exe, expected_fail : true)

test cases/failing test/7 success returncode/failing.c

@@ -0,0 +1,3 @@
+int main(void) {
+    return 0;
+}

test cases/failing test/7 success returncode/meson.build

@@ -0,0 +1,4 @@
+project('success returncode', 'c')
+
+exe = executable('prog', 'failing.c')
+test('errorcode', exe, success_returncode: 1)