From 5e90919912e3040944b06d384f2cd9c0196a0f02 Mon Sep 17 00:00:00 2001 From: Simon Lecoq <22963968+lowlighter@users.noreply.github.com> Date: Mon, 13 Dec 2021 16:48:54 +0100 Subject: [PATCH] feat: `Must` assertions support (#463) * feat: add support for must assertions Signed-off-by: GitHub --- README.md | 17 +++++++++++++ assertion.go | 25 +++++++++++++------ process_testcase.go | 7 ++++++ process_testsuite.go | 8 ++++++ tests/failing/must_assertion.yml | 3 +++ .../foobar_custom_must_assertion.yml | 8 ++++++ tests/user_executor_custom_must_assertion.yml | 13 ++++++++++ types.go | 1 + 8 files changed, 75 insertions(+), 7 deletions(-) create mode 100644 tests/failing/must_assertion.yml create mode 100644 tests/lib_custom/foobar_custom_must_assertion.yml create mode 100644 tests/user_executor_custom_must_assertion.yml diff --git a/README.md b/README.md index 381a255b..43f8a0ef 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ It can also generate xUnit result files. * [Tests Report](#tests-report) * [Assertion](#assertion) * [Keywords](#keywords) + * [`Must` Keywords](#must-keywords) * [Advanced usage](#advanced-usage) * [Debug your testsuites](#debug-your-testsuites) * [Skip testcase](#skip-testcase) @@ -376,6 +377,8 @@ Builtin variables: * {{.venom.teststep.number}} * {{.venom.datetime}} * {{.venom.timestamp}} +* {{.venom.executable}} +* {{.venom.libdir}} # Tests Report @@ -431,6 +434,20 @@ Available formats: jUnit (xml), json, yaml, tap reports * ShouldHappenOnOrAfter - [example](https://github.com/ovh/venom/tree/master/tests/assertions/ShouldHappenOnOrAfter.yml) * ShouldHappenBetween - [example](https://github.com/ovh/venom/tree/master/tests/assertions/ShouldHappenBetween.yml) +### `Must` keywords + +All the above assertions keywords also have a `Must` counterpart which can be used to create a required passing assertion and prevent test cases (and custom executors) to run remaining steps. + +Example: +```yml +- steps: + - type: exec + script: exit 1 + assertions: + - result.code MustEqual 0 + # Remaining steps in this context will not be executed +``` + ## Using logical operators While assertions use `and` operator implicitely, it is possible to use other logical operators to perform complex assertions. diff --git a/assertion.go b/assertion.go index 635c2750..d0a9f214 100644 --- a/assertion.go +++ b/assertion.go @@ -71,9 +71,10 @@ func applyAssertions(r interface{}, tc TestCase, stepNumber int, step TestStep, } type assertion struct { - Actual interface{} - Func assertions.AssertFunc - Args []interface{} + Actual interface{} + Func assertions.AssertFunc + Args []interface{} + Required bool } func parseAssertions(ctx context.Context, s string, input interface{}) (*assertion, error) { @@ -87,6 +88,13 @@ func parseAssertions(ctx context.Context, s string, input interface{}) (*asserti } actual := dump[assert[0]] + // "Must" assertions use same tests as "Should" ones, only the flag changes + required := false + if strings.HasPrefix(assert[1], "Must") { + required = true + assert[1] = strings.Replace(assert[1], "Must", "Should", 1) + } + f, ok := assertions.Get(assert[1]) if !ok { return nil, errors.New("assertion not supported") @@ -100,9 +108,10 @@ func parseAssertions(ctx context.Context, s string, input interface{}) (*asserti } } return &assertion{ - Actual: actual, - Func: f, - Args: args, + Actual: actual, + Func: f, + Args: args, + Required: required, }, nil } @@ -205,7 +214,9 @@ func checkString(tc TestCase, stepNumber int, assertion string, r interface{}) ( } if err := assert.Func(assert.Actual, assert.Args...); err != nil { - return nil, newFailure(tc, stepNumber, assertion, err) + failure := newFailure(tc, stepNumber, assertion, err) + failure.AssertionRequired = assert.Required + return nil, failure } return nil, nil } diff --git a/process_testcase.go b/process_testcase.go index 44234b0f..e2fc4ee5 100644 --- a/process_testcase.go +++ b/process_testcase.go @@ -256,9 +256,11 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase) { tc.testSteps = append(tc.testSteps, step) var hasFailed bool + var isRequired bool if len(tc.Failures) > 0 { for _, f := range tc.Failures { Warning(ctx, "%v", f) + isRequired = isRequired || f.AssertionRequired } hasFailed = true } @@ -272,6 +274,11 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase) { } if hasFailed { + if isRequired { + failure := newFailure(*tc, stepNumber, "", fmt.Errorf("At least one required assertion failed, skipping remaining steps")) + tc.Failures = append(tc.Failures, *failure) + return + } break } diff --git a/process_testsuite.go b/process_testsuite.go index a684d7ea..911809f3 100644 --- a/process_testsuite.go +++ b/process_testsuite.go @@ -44,6 +44,14 @@ func (v *Venom) runTestSuite(ctx context.Context, ts *TestSuite) { ts.Vars.Add(k, computedV) } + exePath, err := os.Executable() + if err != nil { + log.Errorf("failed to get executable path: %v", err) + } else { + ts.Vars.Add("venom.executable", exePath) + } + + ts.Vars.Add("venom.libdir", v.LibDir) ts.Vars.Add("venom.testsuite", ts.Name) ts.ComputedVars = H{} diff --git a/tests/failing/must_assertion.yml b/tests/failing/must_assertion.yml new file mode 100644 index 00000000..2613723c --- /dev/null +++ b/tests/failing/must_assertion.yml @@ -0,0 +1,3 @@ +testcases: +- steps: + - type: foobarcustommustassertion \ No newline at end of file diff --git a/tests/lib_custom/foobar_custom_must_assertion.yml b/tests/lib_custom/foobar_custom_must_assertion.yml new file mode 100644 index 00000000..7aa38a6e --- /dev/null +++ b/tests/lib_custom/foobar_custom_must_assertion.yml @@ -0,0 +1,8 @@ +executor: foobarcustommustassertion +steps: +- script: exit 1 + assertions: + - result.code MustEqual 0 +- script: exit 0 + assertions: + - result.code MustEqual 0 diff --git a/tests/user_executor_custom_must_assertion.yml b/tests/user_executor_custom_must_assertion.yml new file mode 100644 index 00000000..d80164a2 --- /dev/null +++ b/tests/user_executor_custom_must_assertion.yml @@ -0,0 +1,13 @@ +name: testsuite with a user executor in custom dir which has multiple steps and a must assertion +testcases: +- name: testfoobar multisteps custom with a must assertion + steps: + # spawn a venom sub-process and expect it to fail and make assertions on its error messages + - type: exec + script: '{{.venom.executable}} run failing/must_assertion.yml --lib-dir {{.venom.libdir}}' + assertions: + - result.code ShouldEqual 2 + - result.systemout ShouldContainSubstring 'At least one required assertion failed, skipping remaining steps' + - result.systemout ShouldContainSubstring '0:' # matches step #0: + - result.systemout ShouldNotContainSubstring '1:' # matches step #1: + - result.systemerr ShouldBeEmpty diff --git a/types.go b/types.go index c9ae5c07..cae5e30d 100644 --- a/types.go +++ b/types.go @@ -183,6 +183,7 @@ type Failure struct { TestcaseLineNumber int `xml:"-" json:"-" yaml:"-"` StepNumber int `xml:"-" json:"-" yaml:"-"` Assertion string `xml:"-" json:"-" yaml:"-"` + AssertionRequired bool `xml:"-" json:"-" yaml:"-"` Error error `xml:"-" json:"-" yaml:"-"` Value string `xml:",cdata" json:"value" yaml:"value,omitempty"`