feat(test): step.fail and step.fixme modifiers (#34042)

docs/src/test-api/class-test.md

@@ -1773,6 +1773,112 @@ Specifies a custom location for the step to be shown in test reports and trace v
 
 Maximum time in milliseconds for the step to finish. Defaults to `0` (no timeout).
 
+## async method: Test.step.fail
+* since: v1.50
+- returns: <[void]>
+
+Marks a test step as "should fail". Playwright runs this test step and ensures that it actually fails. This is useful for documentation purposes to acknowledge that some functionality is broken until it is fixed.
+
+:::note
+If the step exceeds the timeout, a [TimeoutError] is thrown. This indicates the step did not fail as expected.
+:::
+
+**Usage**
+
+You can declare a test step as failing, so that Playwright ensures it actually fails.
+
+```js
+import { test, expect } from '@playwright/test';
+
+test('my test', async ({ page }) => {
+  // ...
+  await test.step.fail('currently failing', async () => {
+    // ...
+  });
+});
+```
+
+### param: Test.step.fail.title
+* since: v1.50
+- `title` <[string]>
+
+Step name.
+
+### param: Test.step.fail.body
+* since: v1.50
+- `body` <[function]\(\):[Promise]<[any]>>
+
+Step body.
+
+### option: Test.step.fail.box
+* since: v1.50
+- `box` <boolean>
+
+Whether to box the step in the report. Defaults to `false`. When the step is boxed, errors thrown from the step internals point to the step call site. See below for more details.
+
+### option: Test.step.fail.location
+* since: v1.50
+- `location` <[Location]>
+
+Specifies a custom location for the step to be shown in test reports and trace viewer. By default, location of the [`method: Test.step`] call is shown.
+
+### option: Test.step.fail.timeout
+* since: v1.50
+- `timeout` <[float]>
+
+Maximum time in milliseconds for the step to finish. Defaults to `0` (no timeout).
+
+## async method: Test.step.fixme
+* since: v1.50
+- returns: <[void]>
+
+Mark a test step as "fixme", with the intention to fix it. Playwright will not run the step.
+
+**Usage**
+
+You can declare a test step as failing, so that Playwright ensures it actually fails.
+
+```js
+import { test, expect } from '@playwright/test';
+
+test('my test', async ({ page }) => {
+  // ...
+  await test.step.fixme('not yet ready', async () => {
+    // ...
+  });
+});
+```
+
+### param: Test.step.fixme.title
+* since: v1.50
+- `title` <[string]>
+
+Step name.
+
+### param: Test.step.fixme.body
+* since: v1.50
+- `body` <[function]\(\):[Promise]<[any]>>
+
+Step body.
+
+### option: Test.step.fixme.box
+* since: v1.50
+- `box` <boolean>
+
+Whether to box the step in the report. Defaults to `false`. When the step is boxed, errors thrown from the step internals point to the step call site. See below for more details.
+
+### option: Test.step.fixme.location
+* since: v1.50
+- `location` <[Location]>
+
+Specifies a custom location for the step to be shown in test reports and trace viewer. By default, location of the [`method: Test.step`] call is shown.
+
+### option: Test.step.fixme.timeout
+* since: v1.50
+- `timeout` <[float]>
+
+Maximum time in milliseconds for the step to finish. Defaults to `0` (no timeout).
+
 ## method: Test.use
 * since: v1.10
 

packages/playwright/src/common/testType.ts

@@ -56,7 +56,9 @@ export class TestTypeImpl {
     test.fail.only = wrapFunctionWithLocation(this._createTest.bind(this, 'fail.only'));
     test.slow = wrapFunctionWithLocation(this._modifier.bind(this, 'slow'));
     test.setTimeout = wrapFunctionWithLocation(this._setTimeout.bind(this));
-    test.step = this._step.bind(this);
+    test.step = this._step.bind(this, 'pass');
+    test.step.fail = this._step.bind(this, 'fail');
+    test.step.fixme = this._step.bind(this, 'fixme');
     test.use = wrapFunctionWithLocation(this._use.bind(this));
     test.extend = wrapFunctionWithLocation(this._extend.bind(this));
     test.info = () => {
@@ -257,22 +259,40 @@ export class TestTypeImpl {
     suite._use.push({ fixtures, location });
   }
 
-  async _step<T>(title: string, body: () => T | Promise<T>, options: {box?: boolean, location?: Location, timeout?: number } = {}): Promise<T> {
+  async _step<T>(expectation: 'pass'|'fail'|'fixme', title: string, body: () => T | Promise<T>, options: {box?: boolean, location?: Location, timeout?: number } = {}): Promise<T> {
     const testInfo = currentTestInfo();
     if (!testInfo)
       throw new Error(`test.step() can only be called from a test`);
+    if (expectation === 'fixme')
+      return undefined as T;
     const step = testInfo._addStep({ category: 'test.step', title, location: options.location, box: options.box });
     return await zones.run('stepZone', step, async () => {
+      let result;
+      let error;
       try {
-        const result = await raceAgainstDeadline(async () => body(), options.timeout ? monotonicTime() + options.timeout : 0);
-        if (result.timedOut)
-          throw new errors.TimeoutError(`Step timeout ${options.timeout}ms exceeded.`);
-        step.complete({});
-        return result.result;
-      } catch (error) {
+        result = await raceAgainstDeadline(async () => body(), options.timeout ? monotonicTime() + options.timeout : 0);
+      } catch (e) {
+        error = e;
+      }
+      if (result?.timedOut) {
+        const error = new errors.TimeoutError(`Step timeout ${options.timeout}ms exceeded.`);
+        step.complete({ error });
+        throw error;
+      }
+      const expectedToFail = expectation === 'fail';
+      if (error) {
+        step.complete({ error });
+        if (expectedToFail)
+          return undefined as T;
+        throw error;
+      }
+      if (expectedToFail) {
+        error = new Error(`Step is expected to fail, but passed`);
         step.complete({ error });
         throw error;
       }
+      step.complete({});
+      return result!.result;
     });
   }
 

packages/playwright/types/test.d.ts

@@ -5551,7 +5551,217 @@ export interface TestType<TestArgs extends {}, WorkerArgs extends {}> {
    * @param body Step body.
    * @param options
    */
-  step<T>(title: string, body: () => T | Promise<T>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<T>;
+  step: {
+    /**
+   * Declares a test step that is shown in the report.
+   *
+   * **Usage**
+   *
+   * ```js
+   * import { test, expect } from '@playwright/test';
+   *
+   * test('test', async ({ page }) => {
+   *   await test.step('Log in', async () => {
+   *     // ...
+   *   });
+   *
+   *   await test.step('Outer step', async () => {
+   *     // ...
+   *     // You can nest steps inside each other.
+   *     await test.step('Inner step', async () => {
+   *       // ...
+   *     });
+   *   });
+   * });
+   * ```
+   *
+   * **Details**
+   *
+   * The method returns the value returned by the step callback.
+   *
+   * ```js
+   * import { test, expect } from '@playwright/test';
+   *
+   * test('test', async ({ page }) => {
+   *   const user = await test.step('Log in', async () => {
+   *     // ...
+   *     return 'john';
+   *   });
+   *   expect(user).toBe('john');
+   * });
+   * ```
+   *
+   * **Decorator**
+   *
+   * You can use TypeScript method decorators to turn a method into a step. Each call to the decorated method will show
+   * up as a step in the report.
+   *
+   * ```js
+   * function step(target: Function, context: ClassMethodDecoratorContext) {
+   *   return function replacementMethod(...args: any) {
+   *     const name = this.constructor.name + '.' + (context.name as string);
+   *     return test.step(name, async () => {
+   *       return await target.call(this, ...args);
+   *     });
+   *   };
+   * }
+   *
+   * class LoginPage {
+   *   constructor(readonly page: Page) {}
+   *
+   *   @step
+   *   async login() {
+   *     const account = { username: 'Alice', password: 's3cr3t' };
+   *     await this.page.getByLabel('Username or email address').fill(account.username);
+   *     await this.page.getByLabel('Password').fill(account.password);
+   *     await this.page.getByRole('button', { name: 'Sign in' }).click();
+   *     await expect(this.page.getByRole('button', { name: 'View profile and more' })).toBeVisible();
+   *   }
+   * }
+   *
+   * test('example', async ({ page }) => {
+   *   const loginPage = new LoginPage(page);
+   *   await loginPage.login();
+   * });
+   * ```
+   *
+   * **Boxing**
+   *
+   * When something inside a step fails, you would usually see the error pointing to the exact action that failed. For
+   * example, consider the following login step:
+   *
+   * ```js
+   * async function login(page) {
+   *   await test.step('login', async () => {
+   *     const account = { username: 'Alice', password: 's3cr3t' };
+   *     await page.getByLabel('Username or email address').fill(account.username);
+   *     await page.getByLabel('Password').fill(account.password);
+   *     await page.getByRole('button', { name: 'Sign in' }).click();
+   *     await expect(page.getByRole('button', { name: 'View profile and more' })).toBeVisible();
+   *   });
+   * }
+   *
+   * test('example', async ({ page }) => {
+   *   await page.goto('https://github.com/login');
+   *   await login(page);
+   * });
+   * ```
+   *
+   * ```txt
+   * Error: Timed out 5000ms waiting for expect(locator).toBeVisible()
+   *   ... error details omitted ...
+   *
+   *    8 |     await page.getByRole('button', { name: 'Sign in' }).click();
+   * >  9 |     await expect(page.getByRole('button', { name: 'View profile and more' })).toBeVisible();
+   *      |                                                                               ^
+   *   10 |   });
+   * ```
+   *
+   * As we see above, the test may fail with an error pointing inside the step. If you would like the error to highlight
+   * the "login" step instead of its internals, use the `box` option. An error inside a boxed step points to the step
+   * call site.
+   *
+   * ```js
+   * async function login(page) {
+   *   await test.step('login', async () => {
+   *     // ...
+   *   }, { box: true });  // Note the "box" option here.
+   * }
+   * ```
+   *
+   * ```txt
+   * Error: Timed out 5000ms waiting for expect(locator).toBeVisible()
+   *   ... error details omitted ...
+   *
+   *   14 |   await page.goto('https://github.com/login');
+   * > 15 |   await login(page);
+   *      |         ^
+   *   16 | });
+   * ```
+   *
+   * You can also create a TypeScript decorator for a boxed step, similar to a regular step decorator above:
+   *
+   * ```js
+   * function boxedStep(target: Function, context: ClassMethodDecoratorContext) {
+   *   return function replacementMethod(...args: any) {
+   *     const name = this.constructor.name + '.' + (context.name as string);
+   *     return test.step(name, async () => {
+   *       return await target.call(this, ...args);
+   *     }, { box: true });  // Note the "box" option here.
+   *   };
+   * }
+   *
+   * class LoginPage {
+   *   constructor(readonly page: Page) {}
+   *
+   *   @boxedStep
+   *   async login() {
+   *     // ....
+   *   }
+   * }
+   *
+   * test('example', async ({ page }) => {
+   *   const loginPage = new LoginPage(page);
+   *   await loginPage.login();  // <-- Error will be reported on this line.
+   * });
+   * ```
+   *
+   * @param title Step name.
+   * @param body Step body.
+   * @param options
+   */
+  <T>(title: string, body: () => T | Promise<T>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<T>;
+    /**
+   * Mark a test step as "fixme", with the intention to fix it. Playwright will not run the step.
+   *
+   * **Usage**
+   *
+   * You can declare a test step as failing, so that Playwright ensures it actually fails.
+   *
+   * ```js
+   * import { test, expect } from '@playwright/test';
+   *
+   * test('my test', async ({ page }) => {
+   *   // ...
+   *   await test.step.fixme('not yet ready', async () => {
+   *     // ...
+   *   });
+   * });
+   * ```
+   *
+   * @param title Step name.
+   * @param body Step body.
+   * @param options
+   */
+  fixme(title: string, body: () => any | Promise<any>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<void>;
+    /**
+   * Marks a test step as "should fail". Playwright runs this test step and ensures that it actually fails. This is
+   * useful for documentation purposes to acknowledge that some functionality is broken until it is fixed.
+   *
+   * **NOTE** If the step exceeds the timeout, a [TimeoutError](https://playwright.dev/docs/api/class-timeouterror) is
+   * thrown. This indicates the step did not fail as expected.
+   *
+   * **Usage**
+   *
+   * You can declare a test step as failing, so that Playwright ensures it actually fails.
+   *
+   * ```js
+   * import { test, expect } from '@playwright/test';
+   *
+   * test('my test', async ({ page }) => {
+   *   // ...
+   *   await test.step.fail('currently failing', async () => {
+   *     // ...
+   *   });
+   * });
+   * ```
+   *
+   * @param title Step name.
+   * @param body Step body.
+   * @param options
+   */
+  fail(title: string, body: () => any | Promise<any>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<void>;
+  }
   /**
    * `expect` function can be used to create test assertions. Read more about [test assertions](https://playwright.dev/docs/test-assertions).
    *