add additional tests

src/discovery.rs

@@ -160,7 +160,7 @@ fn get_test_methods(file: &Path, tests: &mut Vec<Test>) {
       if name.starts_with("Test") || name.ends_with("Test") || name.ends_with("Tests") {
         for statement in &class_def.body {
           if let Some(method) = statement.as_function_def_stmt() {
-            if method.name.starts_with("test") {
+            if method.name.starts_with("test") || method.name.ends_with("test") {
               tests.push(Test::Method {
                 file: file.into(),
                 class: name.to_string(),

tests/execution.rs

@@ -20,15 +20,6 @@ macro_rules! execution_test {
           .iter()
           .find(|result| &result.test_identifier == test_name)
         else {
-          let cmd_output = Command::cargo_bin(env!("CARGO_PKG_NAME"))
-            .unwrap()
-            .arg(test_file_path)
-            .arg("--output=json")
-            .arg("--no-fail-fast")
-            .output()
-            .unwrap();
-          println!("{cmd_output:?}");
-          println!("expected results {expected_results:?} test results {test_results:?}");
           panic!("Expected test '{test_name}' to have been run");
         };
 
@@ -103,7 +94,7 @@ fn expected_results(test_file: &str) -> HashMap<String, (Outcome, Option<ErrorAs
             "FAIL" => Outcome::Fail,
             "SKIP" => Outcome::Skip,
             "EXPECTED FAILURE" => Outcome::ExpectedFailure,
-            "NON TEST FAIL" => Outcome::NonTestFail,
+            "NON-TEST FAIL" => Outcome::NonTestFail,
             _ => panic!("Unknown Outcome: {}", outcome.trim()),
           },
           error,
@@ -177,10 +168,12 @@ execution_test!(failing_test);
 execution_test!(imports);
 execution_test!(import_submodule, "package/import_submodule");
 execution_test!(import_decimal);
-execution_test!(invalid_code);
+execution_test!(invalid_method);
+execution_test!(invalid_module);
+execution_test!(invalid_remove);
 #[cfg(feature = "ci")] // Takes a long time, so don't want it slowing down developement cycles
 execution_test!(long_running);
-#[cfg(not(feature = "ci"))] // Pytest crashes in CI, with a double free error - don't know why
+#[cfg(any())]
 execution_test!(pytest_marks);
 execution_test!(skip_tests);
 execution_test!(times); // No tests are in this file, just a standard python file

tests/execution/expected_error.py

@@ -3,6 +3,8 @@
 
 - test_will_fail: PASS
 - test_wont_fail: EXPECTED FAILURE
+- TestSuite.method_will_fail_test: PASS
+- TestSuite.method_wont_fail_test: EXPECTED FAILURE
 """
 
 import unittest
@@ -16,3 +18,13 @@ def test_will_fail():
 @unittest.expectedFailure
 def test_wont_fail():
     pass
+
+
+class TestSuite(unittest.TestCase):
+    @unittest.expectedFailure
+    def method_will_fail_test(self):
+        assert False
+
+    @unittest.expectedFailure
+    def method_wont_fail_test(self):
+        assert True

tests/execution/failing_test.py

@@ -3,8 +3,12 @@
 
 - test_regular_fail: FAIL { "kind": "AssertionError" }
 - test_other_error: FAIL { "kind": "TypeError" }
+- TestSuite.test_method_fail: FAIL { "kind": "AssertionError" }
+- TestSuite.test_method_other_error: FAIL { "kind": "TypeError" }
 """
 
+import unittest
+
 
 def test_regular_fail():
     """An assertion fails in this test"""
@@ -15,3 +19,12 @@ def test_other_error():
     """Some other error occurs in this test"""
     variable = 5
     variable[4]
+
+
+class TestSuite(unittest.TestCase):
+    def test_method_fail(self):
+        assert (1 + 3) == 5
+
+    def test_method_other_error(self):
+        variable = 5
+        variable[4]

tests/execution/invalid_code.py → tests/execution/invalid_method.py

@@ -2,6 +2,7 @@
 Code is invalid, so will raise error
 
 - TestSuite.test_missing_self_parameter: FAIL { "kind": "TypeError", "message": "TestSuite.test_missing_self_parameter() takes 0 positional arguments but 1 was given" }
+- BrokenClassTest.test_class_broken: NON-TEST FAIL
 """
 
 import unittest
@@ -18,3 +19,12 @@ def test_missing_self_parameter():
         assert add(1, 2) == 3
         assert add(1, 3) == 4
         assert add(1, 4) == 5
+
+
+class BrokenClassTest(unittest.TestCase):
+    def __init__(self):
+        x = 5
+        x[2]
+
+    def test_class_broken():
+        assert True

tests/execution/invalid_module.py

@@ -0,0 +1,27 @@
+"""
+Raises an error at the module level, so no tests can be run
+
+- test: NON-TEST FAIL
+- TestSuite.test: NON-TEST FAIL
+"""
+
+import unittest
+
+x = 5 + ""
+
+
+def add(a, b):
+    return a + b
+
+
+def test():
+    assert add(1, 2) == 3
+    assert add(1, 3) == 4
+    assert add(1, 4) == 5
+
+
+class TestSuite(unittest.TestCase):
+    def test(self):
+        assert add(1, 2) == 3
+        assert add(1, 3) == 4
+        assert add(1, 4) == 5

tests/execution/invalid_remove.py

@@ -0,0 +1,32 @@
+"""
+Removes/ replaces functions/ classes so they don't exist when they are called at the test stage
+
+- test_function: NON-TEST FAIL
+- TestSuite.test_method: NON-TEST FAIL
+- TestSuiteReplace.test_replaced_method: FAIL { "kind": "TypeError", "message": "'int' object is not callable"}
+"""
+
+import unittest
+
+
+def test_function():
+    assert True
+
+
+del test_function
+
+
+class TestSuite(unittest.TestCase):
+    def test_method(self):
+        assert True
+
+
+del TestSuite
+
+
+class TestSuiteReplace(unittest.TestCase):
+    def __init__(self):
+        setattr(self, "test_replaced_method", 5)
+
+    def test_replaced_method(self):
+        assert True

tests/execution/skip_tests.py

@@ -5,6 +5,7 @@
 - TestSuite.test_skip_method: SKIP
 - TestSuiteTwo.test_skip_class: SKIP
 - skip_by_raising_an_exception_test: SKIP
+- TestSuiteThree.test_skip_by_exception: SKIP
 - test_skip_if_true: SKIP
 - test_skip_if_false: FAIL
 """
@@ -33,6 +34,11 @@ def skip_by_raising_an_exception_test():
     raise unittest.SkipTest("Skipping a function by raising")
 
 
+class TestSuiteThree(unittest.TestCase):
+    def test_skip_by_exception(self):
+        raise unittest.SkipTest("Skipping a function by raising")
+
+
 @unittest.skipIf(True, reason="To test skipping")
 def test_skip_if_true():
     assert False