From 7ed60ccf7f34fbe363b892d463c5e0184e921a2a Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Tue, 17 Dec 2024 11:17:22 -0800 Subject: [PATCH] feat(test): step.fail and step.fixme modifiers (#34042) --- docs/src/test-api/class-test.md | 106 +++++++++++ packages/playwright/src/common/testType.ts | 36 +++- packages/playwright/types/test.d.ts | 212 ++++++++++++++++++++- tests/playwright-test/test-step.spec.ts | 100 ++++++++++ tests/playwright-test/types-2.spec.ts | 23 +++ utils/generate_types/overrides-test.d.ts | 6 +- 6 files changed, 473 insertions(+), 10 deletions(-) diff --git a/docs/src/test-api/class-test.md b/docs/src/test-api/class-test.md index 77a11c073f3f2..d6f1d875130ac 100644 --- a/docs/src/test-api/class-test.md +++ b/docs/src/test-api/class-test.md @@ -1773,6 +1773,112 @@ Specifies a custom location for the step to be shown in test reports and trace v Maximum time in milliseconds for the step to finish. Defaults to `0` (no timeout). +## async method: Test.step.fail +* since: v1.50 +- returns: <[void]> + +Marks a test step as "should fail". Playwright runs this test step and ensures that it actually fails. This is useful for documentation purposes to acknowledge that some functionality is broken until it is fixed. + +:::note +If the step exceeds the timeout, a [TimeoutError] is thrown. This indicates the step did not fail as expected. +::: + +**Usage** + +You can declare a test step as failing, so that Playwright ensures it actually fails. + +```js +import { test, expect } from '@playwright/test'; + +test('my test', async ({ page }) => { + // ... + await test.step.fail('currently failing', async () => { + // ... + }); +}); +``` + +### param: Test.step.fail.title +* since: v1.50 +- `title` <[string]> + +Step name. + +### param: Test.step.fail.body +* since: v1.50 +- `body` <[function]\(\):[Promise]<[any]>> + +Step body. + +### option: Test.step.fail.box +* since: v1.50 +- `box` + +Whether to box the step in the report. Defaults to `false`. When the step is boxed, errors thrown from the step internals point to the step call site. See below for more details. + +### option: Test.step.fail.location +* since: v1.50 +- `location` <[Location]> + +Specifies a custom location for the step to be shown in test reports and trace viewer. By default, location of the [`method: Test.step`] call is shown. + +### option: Test.step.fail.timeout +* since: v1.50 +- `timeout` <[float]> + +Maximum time in milliseconds for the step to finish. Defaults to `0` (no timeout). + +## async method: Test.step.fixme +* since: v1.50 +- returns: <[void]> + +Mark a test step as "fixme", with the intention to fix it. Playwright will not run the step. + +**Usage** + +You can declare a test step as failing, so that Playwright ensures it actually fails. + +```js +import { test, expect } from '@playwright/test'; + +test('my test', async ({ page }) => { + // ... + await test.step.fixme('not yet ready', async () => { + // ... + }); +}); +``` + +### param: Test.step.fixme.title +* since: v1.50 +- `title` <[string]> + +Step name. + +### param: Test.step.fixme.body +* since: v1.50 +- `body` <[function]\(\):[Promise]<[any]>> + +Step body. + +### option: Test.step.fixme.box +* since: v1.50 +- `box` + +Whether to box the step in the report. Defaults to `false`. When the step is boxed, errors thrown from the step internals point to the step call site. See below for more details. + +### option: Test.step.fixme.location +* since: v1.50 +- `location` <[Location]> + +Specifies a custom location for the step to be shown in test reports and trace viewer. By default, location of the [`method: Test.step`] call is shown. + +### option: Test.step.fixme.timeout +* since: v1.50 +- `timeout` <[float]> + +Maximum time in milliseconds for the step to finish. Defaults to `0` (no timeout). + ## method: Test.use * since: v1.10 diff --git a/packages/playwright/src/common/testType.ts b/packages/playwright/src/common/testType.ts index d3c2f1c23a95d..61f9b3682438d 100644 --- a/packages/playwright/src/common/testType.ts +++ b/packages/playwright/src/common/testType.ts @@ -56,7 +56,9 @@ export class TestTypeImpl { test.fail.only = wrapFunctionWithLocation(this._createTest.bind(this, 'fail.only')); test.slow = wrapFunctionWithLocation(this._modifier.bind(this, 'slow')); test.setTimeout = wrapFunctionWithLocation(this._setTimeout.bind(this)); - test.step = this._step.bind(this); + test.step = this._step.bind(this, 'pass'); + test.step.fail = this._step.bind(this, 'fail'); + test.step.fixme = this._step.bind(this, 'fixme'); test.use = wrapFunctionWithLocation(this._use.bind(this)); test.extend = wrapFunctionWithLocation(this._extend.bind(this)); test.info = () => { @@ -257,22 +259,40 @@ export class TestTypeImpl { suite._use.push({ fixtures, location }); } - async _step(title: string, body: () => T | Promise, options: {box?: boolean, location?: Location, timeout?: number } = {}): Promise { + async _step(expectation: 'pass'|'fail'|'fixme', title: string, body: () => T | Promise, options: {box?: boolean, location?: Location, timeout?: number } = {}): Promise { const testInfo = currentTestInfo(); if (!testInfo) throw new Error(`test.step() can only be called from a test`); + if (expectation === 'fixme') + return undefined as T; const step = testInfo._addStep({ category: 'test.step', title, location: options.location, box: options.box }); return await zones.run('stepZone', step, async () => { + let result; + let error; try { - const result = await raceAgainstDeadline(async () => body(), options.timeout ? monotonicTime() + options.timeout : 0); - if (result.timedOut) - throw new errors.TimeoutError(`Step timeout ${options.timeout}ms exceeded.`); - step.complete({}); - return result.result; - } catch (error) { + result = await raceAgainstDeadline(async () => body(), options.timeout ? monotonicTime() + options.timeout : 0); + } catch (e) { + error = e; + } + if (result?.timedOut) { + const error = new errors.TimeoutError(`Step timeout ${options.timeout}ms exceeded.`); + step.complete({ error }); + throw error; + } + const expectedToFail = expectation === 'fail'; + if (error) { + step.complete({ error }); + if (expectedToFail) + return undefined as T; + throw error; + } + if (expectedToFail) { + error = new Error(`Step is expected to fail, but passed`); step.complete({ error }); throw error; } + step.complete({}); + return result!.result; }); } diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 520bcb30d35ff..caed95b8d5673 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -5551,7 +5551,217 @@ export interface TestType { * @param body Step body. * @param options */ - step(title: string, body: () => T | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; + step: { + /** + * Declares a test step that is shown in the report. + * + * **Usage** + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test('test', async ({ page }) => { + * await test.step('Log in', async () => { + * // ... + * }); + * + * await test.step('Outer step', async () => { + * // ... + * // You can nest steps inside each other. + * await test.step('Inner step', async () => { + * // ... + * }); + * }); + * }); + * ``` + * + * **Details** + * + * The method returns the value returned by the step callback. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test('test', async ({ page }) => { + * const user = await test.step('Log in', async () => { + * // ... + * return 'john'; + * }); + * expect(user).toBe('john'); + * }); + * ``` + * + * **Decorator** + * + * You can use TypeScript method decorators to turn a method into a step. Each call to the decorated method will show + * up as a step in the report. + * + * ```js + * function step(target: Function, context: ClassMethodDecoratorContext) { + * return function replacementMethod(...args: any) { + * const name = this.constructor.name + '.' + (context.name as string); + * return test.step(name, async () => { + * return await target.call(this, ...args); + * }); + * }; + * } + * + * class LoginPage { + * constructor(readonly page: Page) {} + * + * @step + * async login() { + * const account = { username: 'Alice', password: 's3cr3t' }; + * await this.page.getByLabel('Username or email address').fill(account.username); + * await this.page.getByLabel('Password').fill(account.password); + * await this.page.getByRole('button', { name: 'Sign in' }).click(); + * await expect(this.page.getByRole('button', { name: 'View profile and more' })).toBeVisible(); + * } + * } + * + * test('example', async ({ page }) => { + * const loginPage = new LoginPage(page); + * await loginPage.login(); + * }); + * ``` + * + * **Boxing** + * + * When something inside a step fails, you would usually see the error pointing to the exact action that failed. For + * example, consider the following login step: + * + * ```js + * async function login(page) { + * await test.step('login', async () => { + * const account = { username: 'Alice', password: 's3cr3t' }; + * await page.getByLabel('Username or email address').fill(account.username); + * await page.getByLabel('Password').fill(account.password); + * await page.getByRole('button', { name: 'Sign in' }).click(); + * await expect(page.getByRole('button', { name: 'View profile and more' })).toBeVisible(); + * }); + * } + * + * test('example', async ({ page }) => { + * await page.goto('https://github.com/login'); + * await login(page); + * }); + * ``` + * + * ```txt + * Error: Timed out 5000ms waiting for expect(locator).toBeVisible() + * ... error details omitted ... + * + * 8 | await page.getByRole('button', { name: 'Sign in' }).click(); + * > 9 | await expect(page.getByRole('button', { name: 'View profile and more' })).toBeVisible(); + * | ^ + * 10 | }); + * ``` + * + * As we see above, the test may fail with an error pointing inside the step. If you would like the error to highlight + * the "login" step instead of its internals, use the `box` option. An error inside a boxed step points to the step + * call site. + * + * ```js + * async function login(page) { + * await test.step('login', async () => { + * // ... + * }, { box: true }); // Note the "box" option here. + * } + * ``` + * + * ```txt + * Error: Timed out 5000ms waiting for expect(locator).toBeVisible() + * ... error details omitted ... + * + * 14 | await page.goto('https://github.com/login'); + * > 15 | await login(page); + * | ^ + * 16 | }); + * ``` + * + * You can also create a TypeScript decorator for a boxed step, similar to a regular step decorator above: + * + * ```js + * function boxedStep(target: Function, context: ClassMethodDecoratorContext) { + * return function replacementMethod(...args: any) { + * const name = this.constructor.name + '.' + (context.name as string); + * return test.step(name, async () => { + * return await target.call(this, ...args); + * }, { box: true }); // Note the "box" option here. + * }; + * } + * + * class LoginPage { + * constructor(readonly page: Page) {} + * + * @boxedStep + * async login() { + * // .... + * } + * } + * + * test('example', async ({ page }) => { + * const loginPage = new LoginPage(page); + * await loginPage.login(); // <-- Error will be reported on this line. + * }); + * ``` + * + * @param title Step name. + * @param body Step body. + * @param options + */ + (title: string, body: () => T | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; + /** + * Mark a test step as "fixme", with the intention to fix it. Playwright will not run the step. + * + * **Usage** + * + * You can declare a test step as failing, so that Playwright ensures it actually fails. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test('my test', async ({ page }) => { + * // ... + * await test.step.fixme('not yet ready', async () => { + * // ... + * }); + * }); + * ``` + * + * @param title Step name. + * @param body Step body. + * @param options + */ + fixme(title: string, body: () => any | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; + /** + * Marks a test step as "should fail". Playwright runs this test step and ensures that it actually fails. This is + * useful for documentation purposes to acknowledge that some functionality is broken until it is fixed. + * + * **NOTE** If the step exceeds the timeout, a [TimeoutError](https://playwright.dev/docs/api/class-timeouterror) is + * thrown. This indicates the step did not fail as expected. + * + * **Usage** + * + * You can declare a test step as failing, so that Playwright ensures it actually fails. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test('my test', async ({ page }) => { + * // ... + * await test.step.fail('currently failing', async () => { + * // ... + * }); + * }); + * ``` + * + * @param title Step name. + * @param body Step body. + * @param options + */ + fail(title: string, body: () => any | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; + } /** * `expect` function can be used to create test assertions. Read more about [test assertions](https://playwright.dev/docs/test-assertions). * diff --git a/tests/playwright-test/test-step.spec.ts b/tests/playwright-test/test-step.spec.ts index ac6845eeae866..74448ccbf8db2 100644 --- a/tests/playwright-test/test-step.spec.ts +++ b/tests/playwright-test/test-step.spec.ts @@ -1494,3 +1494,103 @@ fixture | fixture: context `); }); +test('test.step.fail and test.step.fixme should work', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': stepIndentReporter, + 'playwright.config.ts': `module.exports = { reporter: './reporter' };`, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('test', async ({ }) => { + await test.step('outer step 1', async () => { + await test.step.fail('inner step 1.1', async () => { + throw new Error('inner step 1.1 failed'); + }); + await test.step.fixme('inner step 1.2', async () => {}); + await test.step('inner step 1.3', async () => {}); + }); + await test.step('outer step 2', async () => { + await test.step.fixme('inner step 2.1', async () => {}); + await test.step('inner step 2.2', async () => { + expect(1).toBe(1); + }); + }); + await test.step.fail('outer step 3', async () => { + throw new Error('outer step 3 failed'); + }); + }); + ` + }, { reporter: '' }); + + expect(result.exitCode).toBe(0); + expect(result.report.stats.expected).toBe(1); + expect(result.report.stats.unexpected).toBe(0); + expect(stripAnsi(result.output)).toBe(` +hook |Before Hooks +test.step |outer step 1 @ a.test.ts:4 +test.step | inner step 1.1 @ a.test.ts:5 +test.step | ↪ error: Error: inner step 1.1 failed +test.step | inner step 1.3 @ a.test.ts:9 +test.step |outer step 2 @ a.test.ts:11 +test.step | inner step 2.2 @ a.test.ts:13 +expect | expect.toBe @ a.test.ts:14 +test.step |outer step 3 @ a.test.ts:17 +test.step |↪ error: Error: outer step 3 failed +hook |After Hooks +`); +}); + +test('timeout inside test.step.fail is an error', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': stepIndentReporter, + 'playwright.config.ts': `module.exports = { reporter: './reporter' };`, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('test 2', async ({ }) => { + await test.step('outer step 2', async () => { + await test.step.fail('inner step 2', async () => { + await new Promise(() => {}); + }); + }); + }); + ` + }, { reporter: '', timeout: 2500 }); + + expect(result.exitCode).toBe(1); + expect(result.report.stats.unexpected).toBe(1); + expect(stripAnsi(result.output)).toBe(` +hook |Before Hooks +test.step |outer step 2 @ a.test.ts:4 +test.step | inner step 2 @ a.test.ts:5 +hook |After Hooks +hook |Worker Cleanup + |Test timeout of 2500ms exceeded. +`); +}); + +test('skip test.step.fixme body', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': stepIndentReporter, + 'playwright.config.ts': `module.exports = { reporter: './reporter' };`, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('test', async ({ }) => { + let didRun = false; + await test.step('outer step 2', async () => { + await test.step.fixme('inner step 2', async () => { + didRun = true; + }); + }); + expect(didRun).toBe(false); + }); + ` + }, { reporter: '' }); + + expect(result.exitCode).toBe(0); + expect(result.report.stats.expected).toBe(1); + expect(stripAnsi(result.output)).toBe(` +hook |Before Hooks +test.step |outer step 2 @ a.test.ts:5 +expect |expect.toBe @ a.test.ts:10 +hook |After Hooks +`); +}); diff --git a/tests/playwright-test/types-2.spec.ts b/tests/playwright-test/types-2.spec.ts index f794e06798382..3a06ed0da295b 100644 --- a/tests/playwright-test/types-2.spec.ts +++ b/tests/playwright-test/types-2.spec.ts @@ -204,3 +204,26 @@ test('step should inherit return type from its callback ', async ({ runTSC }) => }); expect(result.exitCode).toBe(0); }); + +test('step.fail and step.fixme return void ', async ({ runTSC }) => { + const result = await runTSC({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('test step.fail', async ({ }) => { + // @ts-expect-error + const bad1: string = await test.step.fail('my step', () => { }); + const good: void = await test.step.fail('my step', async () => { + return 2024; + }); + }); + test('test step.fixme', async ({ }) => { + // @ts-expect-error + const bad1: string = await test.step.fixme('my step', () => { }); + const good: void = await test.step.fixme('my step', async () => { + return 2024; + }); + }); + ` + }); + expect(result.exitCode).toBe(0); +}); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 49c6988ea75d0..3370103a253bd 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -162,7 +162,11 @@ export interface TestType { afterAll(inner: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | any): void; afterAll(title: string, inner: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | any): void; use(fixtures: Fixtures<{}, {}, TestArgs, WorkerArgs>): void; - step(title: string, body: () => T | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; + step: { + (title: string, body: () => T | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; + fixme(title: string, body: () => any | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; + fail(title: string, body: () => any | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; + } expect: Expect<{}>; extend(fixtures: Fixtures): TestType; info(): TestInfo;