fix: overridden tests displayed as failing. (#340)

Signed-off-by: Matteo Pace <[email protected]>
coreruleset · Aug 9, 2024 · f8169f0 · f8169f0
1 parent adb438a
commit f8169f0
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 7 deletions.
diff --git a/runner/run.go b/runner/run.go
@@ -148,9 +148,9 @@ func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase sch
 		return err
 	}
 
-	// Do not even run test if result is overridden. Just use the override and display the overridden result.
+	// Do not even run test if result is overridden. Directly set and display the overridden result.
 	if overridden := overriddenTestResult(ftwCheck, &testCase); overridden != Failed {
-		runContext.Stats.addResultToStats(overridden, &testCase)
+		runContext.Result = overridden
 		displayResult(&testCase, runContext, overridden, time.Duration(0), time.Duration(0))
 		return nil
 	}

diff --git a/runner/run_test.go b/runner/run_test.go
@@ -51,7 +51,7 @@ testoverride:
 	"TestIgnoredTestsRun": `---
 testoverride:
   ignore:
-    ".*-1": "This test result must be ignored"
+    ".*-2": "This test result must be ignored"
   forcefail:
     ".*-8": "This test should pass, but it is going to fail"
   forcepass:
@@ -391,7 +391,10 @@ func (s *runTestSuite) TestFailedTestsRun() {
 func (s *runTestSuite) TestIgnoredTestsRun() {
 	res, err := Run(s.cfg, s.ftwTests, RunnerConfig{}, s.out)
 	s.Require().NoError(err)
-	s.Equal(res.Stats.TotalFailed(), 1, "Oops, test run failed!")
+	s.Equal(1, len(res.Stats.ForcedPass), "Oops, unexpected number of forced pass tests")
+	s.Equal(1, len(res.Stats.Failed), "Oops, unexpected number of failed tests")
+	s.Equal(1, len(res.Stats.ForcedFail), "Oops, unexpected number of forced failed tests")
+	s.Equal(4, len(res.Stats.Ignored), "Oops, unexpected number of ignored tests")
 }
 
 func (s *runTestSuite) TestGetRequestFromTestWithAutocompleteHeaders() {

diff --git a/runner/testdata/TestIgnoredTestsRun.yaml b/runner/testdata/TestIgnoredTestsRun.yaml
@@ -4,6 +4,18 @@ meta:
   description: "Example Test"
 tests:
   - test_id: 1
+    description: "test that fails and is not overridden"
+    stages:
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "none.host"
+        output:
+          status: 413
+  - test_id: 2
     description: "access real external site"
     stages:
       - input:
@@ -28,7 +40,7 @@ tests:
             Host: "localhost"
         output:
           status: 200
-  - test_id: 10
+  - test_id: 20
     stages:
       - input:
           dest_addr: "{{ .TestAddr }}"
@@ -41,7 +53,7 @@ tests:
             Host: "localhost"
         output:
           response_contains: "Hello, client"
-  - test_id: 101
+  - test_id: 201
     description: "this tests exceptions (connection timeout)"
     stages:
       - input:
@@ -53,7 +65,7 @@ tests:
             Host: "none.host"
         output:
           expect_error: True
-  - test_id: 102
+  - test_id: 202
     description: "this tests exceptions (connection timeout)"
     stages:
       - input:
@@ -66,3 +78,15 @@ tests:
           encoded_request: "UE9TVCAvaW5kZXguaHRtbCBIVFRQLzEuMQ0KSG9zdDogMTkyLjE2OC4xLjIzDQpVc2VyLUFnZW50OiBjdXJsLzcuNDMuMA0KQWNjZXB0OiAqLyoNCkNvbnRlbnQtTGVuZ3RoOiA2NA0KQ29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi94LXd3dy1mb3JtLXVybGVuY29kZWQNCkNvbm5lY3Rpb246IGNsb3NlDQoNCmQ9MTsyOzM7NDs1XG4xO0BTVU0oMSsxKSpjbWR8JyBwb3dlcnNoZWxsIElFWCh3Z2V0IDByLnBlL3ApJ1whQTA7Mw=="
         output:
           expect_error: True
+  - test_id: 99
+    description: "test that fails but will pass because it is overridden"
+    stages:
+      - input:
+          dest_addr: "{{ .TestAddr }}"
+          port: {{ .TestPort }}
+          headers:
+            User-Agent: "ModSecurity CRS 3 Tests"
+            Accept: "*/*"
+            Host: "none.host"
+        output:
+          status: 413