feat: Must assertions support (#463)

* feat: add support for must assertions

Signed-off-by: GitHub <[email protected]>

README.md

@@ -21,6 +21,7 @@ It can also generate xUnit result files.
 * [Tests Report](#tests-report)
 * [Assertion](#assertion)
   * [Keywords](#keywords)
+    * [`Must` Keywords](#must-keywords)
 * [Advanced usage](#advanced-usage)
   * [Debug your testsuites](#debug-your-testsuites)
   * [Skip testcase](#skip-testcase)
@@ -376,6 +377,8 @@ Builtin variables:
 * {{.venom.teststep.number}}
 * {{.venom.datetime}}
 * {{.venom.timestamp}}
+* {{.venom.executable}}
+* {{.venom.libdir}}
 
 # Tests Report
 
@@ -431,6 +434,20 @@ Available formats: jUnit (xml), json, yaml, tap reports
 * ShouldHappenOnOrAfter - [example](https://github.com/ovh/venom/tree/master/tests/assertions/ShouldHappenOnOrAfter.yml)
 * ShouldHappenBetween - [example](https://github.com/ovh/venom/tree/master/tests/assertions/ShouldHappenBetween.yml)
 
+### `Must` keywords
+
+All the above assertions keywords also have a `Must` counterpart which can be used to create a required passing assertion and prevent test cases (and custom executors) to run remaining steps.
+
+Example:
+```yml
+- steps:
+  - type: exec
+    script: exit 1
+    assertions:
+      - result.code MustEqual 0
+  # Remaining steps in this context will not be executed
+```
+
 ## Using logical operators
 
 While assertions use `and` operator implicitely, it is possible to use other logical operators to perform complex assertions.

assertion.go

@@ -71,9 +71,10 @@ func applyAssertions(r interface{}, tc TestCase, stepNumber int, step TestStep,
 }
 
 type assertion struct {
-	Actual interface{}
-	Func   assertions.AssertFunc
-	Args   []interface{}
+	Actual   interface{}
+	Func     assertions.AssertFunc
+	Args     []interface{}
+	Required bool
 }
 
 func parseAssertions(ctx context.Context, s string, input interface{}) (*assertion, error) {
@@ -87,6 +88,13 @@ func parseAssertions(ctx context.Context, s string, input interface{}) (*asserti
 	}
 	actual := dump[assert[0]]
 
+	// "Must" assertions use same tests as "Should" ones, only the flag changes
+	required := false
+	if strings.HasPrefix(assert[1], "Must") {
+		required = true
+		assert[1] = strings.Replace(assert[1], "Must", "Should", 1)
+	}
+
 	f, ok := assertions.Get(assert[1])
 	if !ok {
 		return nil, errors.New("assertion not supported")
@@ -100,9 +108,10 @@ func parseAssertions(ctx context.Context, s string, input interface{}) (*asserti
 		}
 	}
 	return &assertion{
-		Actual: actual,
-		Func:   f,
-		Args:   args,
+		Actual:   actual,
+		Func:     f,
+		Args:     args,
+		Required: required,
 	}, nil
 }
 
@@ -205,7 +214,9 @@ func checkString(tc TestCase, stepNumber int, assertion string, r interface{}) (
 	}
 
 	if err := assert.Func(assert.Actual, assert.Args...); err != nil {
-		return nil, newFailure(tc, stepNumber, assertion, err)
+		failure := newFailure(tc, stepNumber, assertion, err)
+		failure.AssertionRequired = assert.Required
+		return nil, failure
 	}
 	return nil, nil
 }

process_testcase.go

@@ -256,9 +256,11 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase) {
 			tc.testSteps = append(tc.testSteps, step)
 
 			var hasFailed bool
+			var isRequired bool
 			if len(tc.Failures) > 0 {
 				for _, f := range tc.Failures {
 					Warning(ctx, "%v", f)
+					isRequired = isRequired || f.AssertionRequired
 				}
 				hasFailed = true
 			}
@@ -272,6 +274,11 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase) {
 			}
 
 			if hasFailed {
+				if isRequired {
+					failure := newFailure(*tc, stepNumber, "", fmt.Errorf("At least one required assertion failed, skipping remaining steps"))
+					tc.Failures = append(tc.Failures, *failure)
+					return
+				}
 				break
 			}
 

process_testsuite.go

@@ -44,6 +44,14 @@ func (v *Venom) runTestSuite(ctx context.Context, ts *TestSuite) {
 		ts.Vars.Add(k, computedV)
 	}
 
+	exePath, err := os.Executable()
+	if err != nil {
+		log.Errorf("failed to get executable path: %v", err)
+	} else {
+		ts.Vars.Add("venom.executable", exePath)
+	}
+
+	ts.Vars.Add("venom.libdir", v.LibDir)
 	ts.Vars.Add("venom.testsuite", ts.Name)
 	ts.ComputedVars = H{}
 

tests/failing/must_assertion.yml

@@ -0,0 +1,3 @@
+testcases:
+- steps:
+  - type: foobarcustommustassertion

tests/lib_custom/foobar_custom_must_assertion.yml

@@ -0,0 +1,8 @@
+executor: foobarcustommustassertion
+steps:
+- script: exit 1
+  assertions: 
+  - result.code MustEqual 0
+- script: exit 0
+  assertions: 
+  - result.code MustEqual 0

tests/user_executor_custom_must_assertion.yml

@@ -0,0 +1,13 @@
+name: testsuite with a user executor in custom dir which has multiple steps and a must assertion
+testcases:
+- name: testfoobar multisteps custom with a must assertion
+  steps:
+  # spawn a venom sub-process and expect it to fail and make assertions on its error messages
+  - type: exec
+    script: '{{.venom.executable}} run failing/must_assertion.yml --lib-dir {{.venom.libdir}}'
+    assertions:
+      - result.code ShouldEqual 2
+      - result.systemout ShouldContainSubstring 'At least one required assertion failed, skipping remaining steps'
+      - result.systemout ShouldContainSubstring '0:' # matches step #0:
+      - result.systemout ShouldNotContainSubstring '1:' # matches step #1:
+      - result.systemerr ShouldBeEmpty

types.go

@@ -183,6 +183,7 @@ type Failure struct {
 	TestcaseLineNumber int    `xml:"-" json:"-" yaml:"-"`
 	StepNumber         int    `xml:"-" json:"-" yaml:"-"`
 	Assertion          string `xml:"-" json:"-" yaml:"-"`
+	AssertionRequired  bool   `xml:"-" json:"-" yaml:"-"`
 	Error              error  `xml:"-" json:"-" yaml:"-"`
 
 	Value   string `xml:",cdata" json:"value" yaml:"value,omitempty"`