From 49b293002cecd4d17acae5197e676101849c2b8c Mon Sep 17 00:00:00 2001 From: BeeMargarida Date: Sat, 6 Apr 2024 18:07:59 +0100 Subject: [PATCH 1/5] fix(playwright): test unexpected result if failed and skipped Checks if a test results are only failed and skipped. If so, it sets the result as unexpected instead of flaky, since they never passed. Fixes #28322 --- packages/playwright/src/common/test.ts | 4 +++ tests/playwright-test/test-serial.spec.ts | 34 +++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts index 5f22f8fe3b05c..3acbfa1b5667f 100644 --- a/packages/playwright/src/common/test.ts +++ b/packages/playwright/src/common/test.ts @@ -290,10 +290,14 @@ export class TestCase extends Base implements reporterTypes.TestCase { return 'skipped'; const failures = results.filter(result => result.status !== 'skipped' && result.status !== 'interrupted' && result.status !== this.expectedStatus); + const skipped = results.filter(result => result.status === 'skipped'); + const passed = results.filter(result => result.status === 'passed'); if (!failures.length) // all passed return 'expected'; if (failures.length === results.length) // all failed return 'unexpected'; + if (failures.length && skipped.length && !passed.length) // some failed, none succedded and the rest where skipped + return 'unexpected'; return 'flaky'; // mixed bag } diff --git a/tests/playwright-test/test-serial.spec.ts b/tests/playwright-test/test-serial.spec.ts index 085d175097028..0e235edcad293 100644 --- a/tests/playwright-test/test-serial.spec.ts +++ b/tests/playwright-test/test-serial.spec.ts @@ -140,6 +140,40 @@ test('test.describe.serial should work with retry', async ({ runInlineTest }) => ]); }); +test('test.describe.serial should work with retry when it fails in different places', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test.describe.serial('serial suite', () => { + test('test1', async ({}, testInfo) => { + console.log('\\n%%test1'); + expect(testInfo.retry).toEqual(0) + }); + test('test2', async ({}, testInfo) => { + console.log('\\n%%test2'); + expect(testInfo.retry).toEqual(1) + }); + test('test3', async ({}) => { + console.log('\\n%%test3'); + }); + test('test4', async ({}) => { + console.log('\\n%%test4'); + }); + }); + `, + }, { retries: 1 }); + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(0); + expect(result.flaky).toBe(1); + expect(result.failed).toBe(1); + expect(result.skipped).toBe(2); + expect(result.outputLines).toEqual([ + 'test1', + 'test2', + 'test1', + ]); +}); + test('test.describe.serial should work with retry and beforeAll failure', async ({ runInlineTest }) => { const result = await runInlineTest({ 'a.test.ts': ` From 3436a862054fce372c6c5304a15f2c909dc91f28 Mon Sep 17 00:00:00 2001 From: BeeMargarida Date: Sat, 6 Apr 2024 18:47:34 +0100 Subject: [PATCH 2/5] fix(playwright): add check for unexpected skips in failure state --- packages/playwright/src/common/test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts index 3acbfa1b5667f..bfa50f6d00b8c 100644 --- a/packages/playwright/src/common/test.ts +++ b/packages/playwright/src/common/test.ts @@ -290,7 +290,7 @@ export class TestCase extends Base implements reporterTypes.TestCase { return 'skipped'; const failures = results.filter(result => result.status !== 'skipped' && result.status !== 'interrupted' && result.status !== this.expectedStatus); - const skipped = results.filter(result => result.status === 'skipped'); + const skipped = results.filter(result => result.status === 'skipped' && result.status !== this.expectedStatus); const passed = results.filter(result => result.status === 'passed'); if (!failures.length) // all passed return 'expected'; From 7eccbf7ff14988deaf28c9ff7be3caab2eb20155 Mon Sep 17 00:00:00 2001 From: Ana Margarida Silva Date: Mon, 8 Apr 2024 12:03:00 +0100 Subject: [PATCH 3/5] chore(playwright): fix typo in comments --- packages/playwright/src/common/test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts index bfa50f6d00b8c..806c74a12e713 100644 --- a/packages/playwright/src/common/test.ts +++ b/packages/playwright/src/common/test.ts @@ -296,7 +296,7 @@ export class TestCase extends Base implements reporterTypes.TestCase { return 'expected'; if (failures.length === results.length) // all failed return 'unexpected'; - if (failures.length && skipped.length && !passed.length) // some failed, none succedded and the rest where skipped + if (failures.length && skipped.length && !passed.length) // some failed, none succeeded and the rest were skipped return 'unexpected'; return 'flaky'; // mixed bag } From c19ebab1223d6411f76ba5b6c652f941719036e9 Mon Sep 17 00:00:00 2001 From: Ana Margarida Silva Date: Thu, 18 Apr 2024 10:11:44 +0100 Subject: [PATCH 4/5] fix(playwright): do not check for skipped tests --- packages/playwright/src/common/test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts index 806c74a12e713..4b3495a477459 100644 --- a/packages/playwright/src/common/test.ts +++ b/packages/playwright/src/common/test.ts @@ -296,7 +296,7 @@ export class TestCase extends Base implements reporterTypes.TestCase { return 'expected'; if (failures.length === results.length) // all failed return 'unexpected'; - if (failures.length && skipped.length && !passed.length) // some failed, none succeeded and the rest were skipped + if (failures.length && !passed.length) // some failed, none succeeded return 'unexpected'; return 'flaky'; // mixed bag } From ad586d27b513ef1970961852da361845fdd99039 Mon Sep 17 00:00:00 2001 From: Ana Margarida Silva Date: Thu, 18 Apr 2024 10:14:47 +0100 Subject: [PATCH 5/5] fix(playwright): use expected status --- packages/playwright/src/common/test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts index 4b3495a477459..e6372326d0cc4 100644 --- a/packages/playwright/src/common/test.ts +++ b/packages/playwright/src/common/test.ts @@ -290,8 +290,7 @@ export class TestCase extends Base implements reporterTypes.TestCase { return 'skipped'; const failures = results.filter(result => result.status !== 'skipped' && result.status !== 'interrupted' && result.status !== this.expectedStatus); - const skipped = results.filter(result => result.status === 'skipped' && result.status !== this.expectedStatus); - const passed = results.filter(result => result.status === 'passed'); + const passed = results.filter(result => result.status === this.expectedStatus); if (!failures.length) // all passed return 'expected'; if (failures.length === results.length) // all failed