From 5bfe560b60a3e6744169bc3bc1fa65bafd3cefbd Mon Sep 17 00:00:00 2001 From: Pierre-Hugues Laune Date: Mon, 23 Dec 2024 17:56:25 +0100 Subject: [PATCH] feat: ignore flaky tests --- README.md | 4 ++ action.js | 3 +- action.test.fixtures.js | 13 +++- action.test.js | 20 ++++++- action.yml | 4 ++ integration-tests/maven/flakes/pom.xml | 60 +++++++++++++++++++ .../report/calc/AllOkWithFlakesTest.java | 20 +++++++ integration-tests/maven/pom.xml | 1 + utils.js | 8 +-- 9 files changed, 126 insertions(+), 7 deletions(-) create mode 100644 integration-tests/maven/flakes/pom.xml create mode 100644 integration-tests/maven/flakes/src/test/java/action/surefire/report/calc/AllOkWithFlakesTest.java diff --git a/README.md b/README.md index 915ec5d4..7535456b 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,10 @@ Optional. Check will fail if there are test failures. The default is `false`. Optional. Check will fail if no tests were found. The default is `true`. +### `ignore_flaky_tests` + +Optional. Set to `true` to consider flaky tests as success. The default is `false`. + ### `skip_publishing` Optional. Skip the test report publishing (check run creation). The default is `false`. diff --git a/action.js b/action.js index 2d38b21c..20d6a039 100644 --- a/action.js +++ b/action.js @@ -15,11 +15,12 @@ const action = async () => { const commit = core.getInput('commit'); const failOnFailedTests = core.getInput('fail_on_test_failures') === 'true'; const failIfNoTests = core.getInput('fail_if_no_tests') === 'true'; + const ignoreFlakyTests = core.getInput('ignore_flaky_tests') === 'true'; const skipPublishing = core.getInput('skip_publishing') === 'true'; const isFilenameInStackTrace = core.getInput('file_name_in_stack_trace') === 'true'; const githubBaseUrl = core.getInput('github_base_url'); - let { count, skipped, annotations } = await parseTestReports(reportPaths, isFilenameInStackTrace); + let { count, skipped, annotations } = await parseTestReports(reportPaths, isFilenameInStackTrace, ignoreFlakyTests); const foundResults = count > 0 || skipped > 0; const conclusion = (foundResults && annotations.length === 0) || (!foundResults && !failIfNoTests) diff --git a/action.test.fixtures.js b/action.test.fixtures.js index c37f92c3..7807ae63 100644 --- a/action.test.fixtures.js +++ b/action.test.fixtures.js @@ -4,7 +4,7 @@ const finishedWithFailures = { status: 'completed', conclusion: 'failure', output: { - title: '19 tests run, 1 skipped, 12 failed.', + title: '20 tests run, 1 skipped, 13 failed.', summary: '', annotations: [ { @@ -112,6 +112,17 @@ const finishedWithFailures = { raw_details: "java.lang.AssertionError: \n\nExpected: \"Good Twin\"\n but: was \"Evil Twin\"\n\tat action.surefire.report.twin.second.TwinTest.should_always_fail(TwinTest.java:13)" }, + { + path: 'integration-tests/maven/flakes/src/test/java/action/surefire/report/calc/AllOkWithFlakesTest.java', + start_line: 1, + end_line: 1, + start_column: 0, + end_column: 0, + annotation_level: 'failure', + title: 'AllOkWithFlakesTest.firstTryFailSecondTrySuccess', + message: 'firstTryFailSecondTrySuccess', + raw_details: "" + }, { path: 'integration-tests/maven/utils/src/test/java/action/surefire/report/calc/CalcUtilsTest.kt', start_line: 27, diff --git a/action.test.js b/action.test.js index 5c1bd979..660233ae 100644 --- a/action.test.js +++ b/action.test.js @@ -107,6 +107,24 @@ describe('action should work', () => { expect(failed).toBeNull(); }); + it('should send all ok if tests were flaky and ignore_flaky_test is true', async () => { + inputs.report_paths = '**/surefire-reports/TEST-*AllOkWithFlakesTest.xml'; + inputs.ignore_flaky_tests = 'true'; + let request = null; + const scope = nock('https://api.github.com') + .post('/repos/scacap/action-surefire-report/check-runs', body => { + request = body; + return body; + }) + .reply(200, {}); + await action(); + scope.done(); + + expect(request).toStrictEqual(finishedSuccess); + expect(outputs).toHaveProperty('conclusion', 'success'); + expect(failed).toBeNull(); + }); + it('should send failure if no test results were found', async () => { inputs.report_paths = '**/xxx/*.xml'; let request = null; @@ -191,7 +209,7 @@ describe('action should work', () => { await action(); scope.done(); - expect(failed).toBe('There were 12 failed tests'); + expect(failed).toBe('There were 13 failed tests'); }); }); diff --git a/action.yml b/action.yml index fa4c673d..01c1da67 100644 --- a/action.yml +++ b/action.yml @@ -31,6 +31,10 @@ inputs: description: 'fail run if there were no test results found' required: false default: 'true' + ignore_flaky_tests: + description: 'consider flaky tests as success' + required: false + default: 'false' skip_publishing: description: 'skip test report publishing' required: false diff --git a/integration-tests/maven/flakes/pom.xml b/integration-tests/maven/flakes/pom.xml new file mode 100644 index 00000000..b4df7c6b --- /dev/null +++ b/integration-tests/maven/flakes/pom.xml @@ -0,0 +1,60 @@ + + + + tests + action.surefire.report + 0.0.1-SNAPSHOT + + 4.0.0 + flakes + + + + junit + junit + test + + + org.hamcrest + hamcrest-all + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + -Duser.language=en + true + true + 1 + + + + org.apache.maven.plugins + maven-compiler-plugin + + + compile + compile + + compile + + + + testCompile + test-compile + + testCompile + + + + + + + + + diff --git a/integration-tests/maven/flakes/src/test/java/action/surefire/report/calc/AllOkWithFlakesTest.java b/integration-tests/maven/flakes/src/test/java/action/surefire/report/calc/AllOkWithFlakesTest.java new file mode 100644 index 00000000..23b34130 --- /dev/null +++ b/integration-tests/maven/flakes/src/test/java/action/surefire/report/calc/AllOkWithFlakesTest.java @@ -0,0 +1,20 @@ +package action.surefire.report.calc; + +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +public class AllOkWithFlakesTest { + + private static boolean failTest = true; + + @Test + public void firstTryFailSecondTrySuccess() { + if(failTest) { + failTest = false; + assertTrue(false); + } else { + assertTrue(true); + } + } +} diff --git a/integration-tests/maven/pom.xml b/integration-tests/maven/pom.xml index 83c9453d..fa5fef3d 100644 --- a/integration-tests/maven/pom.xml +++ b/integration-tests/maven/pom.xml @@ -13,6 +13,7 @@ utils email evil_twins + flakes diff --git a/utils.js b/utils.js index 03d09306..8933b296 100644 --- a/utils.js +++ b/utils.js @@ -62,7 +62,7 @@ function getTestsuites(report) { return [report.testsuites.testsuite]; } -async function parseFile(file, isFilenameInStackTrace) { +async function parseFile(file, isFilenameInStackTrace, ignoreFlakyTests) { core.debug(`Parsing file ${file}`); let count = 0; let skipped = 0; @@ -85,7 +85,7 @@ async function parseFile(file, isFilenameInStackTrace) { for (const testcase of testcases) { count++; if (testcase.skipped) skipped++; - if (testcase.failure || testcase.flakyFailure || testcase.error) { + if (testcase.failure || (testcase.flakyFailure && !ignoreFlakyTests) || testcase.error) { let testcaseData = (testcase.failure && testcase.failure._cdata) || (testcase.failure && testcase.failure._text) || @@ -139,13 +139,13 @@ async function parseFile(file, isFilenameInStackTrace) { return {count, skipped, annotations}; } -const parseTestReports = async (reportPaths, isFilenameInStackTrace) => { +const parseTestReports = async (reportPaths, isFilenameInStackTrace, ignoreFlakyTests) => { const globber = await glob.create(reportPaths, {followSymbolicLinks: false}); let annotations = []; let count = 0; let skipped = 0; for await (const file of globber.globGenerator()) { - const {count: c, skipped: s, annotations: a} = await parseFile(file, isFilenameInStackTrace); + const {count: c, skipped: s, annotations: a} = await parseFile(file, isFilenameInStackTrace, ignoreFlakyTests); if (c === 0) continue; count += c; skipped += s;