feat(testType): add support for test.fail.only method (#33001)

docs/src/test-api/class-test.md

@@ -1138,6 +1138,57 @@ Optional description that will be reflected in a test report.
 
 
 
+## method: Test.fail.only
+* since: v1.49
+
+You can use `test.fail.only` to focus on a specific test that is expected to fail. This is particularly useful when debugging a failing test or working on a specific issue.
+
+To declare a focused "failing" test:
+* `test.fail.only(title, body)`
+* `test.fail.only(title, details, body)`
+
+**Usage**
+
+You can declare a focused failing test, so that Playwright runs only this test and ensures it actually fails.
+
+```js
+import { test, expect } from '@playwright/test';
+
+test.fail.only('focused failing test', async ({ page }) => {
+  // This test is expected to fail
+});
+test('not in the focused group', async ({ page }) => {
+  // This test will not run
+});
+```
+
+### param: Test.fail.only.title
+* since: v1.49
+
+- `title` ?<[string]>
+
+Test title.
+
+### param: Test.fail.only.details
+* since: v1.49
+
+- `details` ?<[Object]>
+  - `tag` ?<[string]|[Array]<[string]>>
+  - `annotation` ?<[Object]|[Array]<[Object]>>
+    - `type` <[string]>
+    - `description` ?<[string]>
+
+See [`method: Test.describe`] for test details description.
+
+### param: Test.fail.only.body
+* since: v1.49
+
+- `body` ?<[function]\([Fixtures], [TestInfo]\)>
+
+Test body that takes one or two arguments: an object with fixtures and optional [TestInfo].
+
+
+
 ## method: Test.fixme
 * since: v1.10
 

packages/playwright/src/common/testType.ts

@@ -52,6 +52,7 @@ export class TestTypeImpl {
     test.skip = wrapFunctionWithLocation(this._modifier.bind(this, 'skip'));
     test.fixme = wrapFunctionWithLocation(this._modifier.bind(this, 'fixme'));
     test.fail = wrapFunctionWithLocation(this._modifier.bind(this, 'fail'));
+    test.fail.only = wrapFunctionWithLocation(this._createTest.bind(this, 'fail.only'));
     test.slow = wrapFunctionWithLocation(this._modifier.bind(this, 'slow'));
     test.setTimeout = wrapFunctionWithLocation(this._setTimeout.bind(this));
     test.step = this._step.bind(this);
@@ -81,7 +82,7 @@ export class TestTypeImpl {
     return suite;
   }
 
-  private _createTest(type: 'default' | 'only' | 'skip' | 'fixme' | 'fail', location: Location, title: string, fnOrDetails: Function | TestDetails, fn?: Function) {
+  private _createTest(type: 'default' | 'only' | 'skip' | 'fixme' | 'fail' | 'fail.only', location: Location, title: string, fnOrDetails: Function | TestDetails, fn?: Function) {
     throwIfRunningInsideJest();
     const suite = this._currentSuite(location, 'test()');
     if (!suite)
@@ -104,10 +105,12 @@ export class TestTypeImpl {
     test._tags.push(...validatedDetails.tags);
     suite._addTest(test);
 
-    if (type === 'only')
+    if (type === 'only' || type === 'fail.only')
       test._only = true;
     if (type === 'skip' || type === 'fixme' || type === 'fail')
       test._staticAnnotations.push({ type });
+    else if (type === 'fail.only')
+      test._staticAnnotations.push({ type: 'fail' });
   }
 
   private _describe(type: 'default' | 'only' | 'serial' | 'serial.only' | 'parallel' | 'parallel.only' | 'skip' | 'fixme', location: Location, titleOrFn: string | Function, fnOrDetails?: TestDetails | Function, fn?: Function) {

packages/playwright/types/test.d.ts

@@ -3625,8 +3625,8 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
    * "should fail" when the return value is `true`.
    * @param description Optional description that will be reflected in a test report.
    */
-  fail(title: string, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise<void> | void): void;
+  fail: {
-  /**
+    /**
    * Marks a test as "should fail". Playwright runs this test and ensures that it is actually failing. This is useful
    * for documentation purposes to acknowledge that some functionality is broken until it is fixed.
    *
@@ -3702,8 +3702,8 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
    * "should fail" when the return value is `true`.
    * @param description Optional description that will be reflected in a test report.
    */
-  fail(title: string, details: TestDetails, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise<void> | void): void;
+  (title: string, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise<void> | void): void;
-  /**
+    /**
    * Marks a test as "should fail". Playwright runs this test and ensures that it is actually failing. This is useful
    * for documentation purposes to acknowledge that some functionality is broken until it is fixed.
    *
@@ -3779,8 +3779,8 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
    * "should fail" when the return value is `true`.
    * @param description Optional description that will be reflected in a test report.
    */
-  fail(condition: boolean, description?: string): void;
+  (title: string, details: TestDetails, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise<void> | void): void;
-  /**
+    /**
    * Marks a test as "should fail". Playwright runs this test and ensures that it is actually failing. This is useful
    * for documentation purposes to acknowledge that some functionality is broken until it is fixed.
    *
@@ -3856,8 +3856,8 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
    * "should fail" when the return value is `true`.
    * @param description Optional description that will be reflected in a test report.
    */
-  fail(callback: (args: TestArgs & WorkerArgs) => boolean, description?: string): void;
+  (condition: boolean, description?: string): void;
-  /**
+    /**
    * Marks a test as "should fail". Playwright runs this test and ensures that it is actually failing. This is useful
    * for documentation purposes to acknowledge that some functionality is broken until it is fixed.
    *
@@ -3933,7 +3933,115 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
    * "should fail" when the return value is `true`.
    * @param description Optional description that will be reflected in a test report.
    */
-  fail(): void;
+  (callback: (args: TestArgs & WorkerArgs) => boolean, description?: string): void;
+    /**
+   * Marks a test as "should fail". Playwright runs this test and ensures that it is actually failing. This is useful
+   * for documentation purposes to acknowledge that some functionality is broken until it is fixed.
+   *
+   * To declare a "failing" test:
+   * - `test.fail(title, body)`
+   * - `test.fail(title, details, body)`
+   *
+   * To annotate test as "failing" at runtime:
+   * - `test.fail(condition, description)`
+   * - `test.fail(callback, description)`
+   * - `test.fail()`
+   *
+   * **Usage**
+   *
+   * You can declare a test as failing, so that Playwright ensures it actually fails.
+   *
+   * ```js
+   * import { test, expect } from '@playwright/test';
+   *
+   * test.fail('not yet ready', async ({ page }) => {
+   *   // ...
+   * });
+   * ```
+   *
+   * If your test fails in some configurations, but not all, you can mark the test as failing inside the test body based
+   * on some condition. We recommend passing a `description` argument in this case.
+   *
+   * ```js
+   * import { test, expect } from '@playwright/test';
+   *
+   * test('fail in WebKit', async ({ page, browserName }) => {
+   *   test.fail(browserName === 'webkit', 'This feature is not implemented for Mac yet');
+   *   // ...
+   * });
+   * ```
+   *
+   * You can mark all tests in a file or
+   * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe) group as
+   * "should fail" based on some condition with a single `test.fail(callback, description)` call.
+   *
+   * ```js
+   * import { test, expect } from '@playwright/test';
+   *
+   * test.fail(({ browserName }) => browserName === 'webkit', 'not implemented yet');
+   *
+   * test('fail in WebKit 1', async ({ page }) => {
+   *   // ...
+   * });
+   * test('fail in WebKit 2', async ({ page }) => {
+   *   // ...
+   * });
+   * ```
+   *
+   * You can also call `test.fail()` without arguments inside the test body to always mark the test as failed. We
+   * recommend declaring a failing test with `test.fail(title, body)` instead.
+   *
+   * ```js
+   * import { test, expect } from '@playwright/test';
+   *
+   * test('less readable', async ({ page }) => {
+   *   test.fail();
+   *   // ...
+   * });
+   * ```
+   *
+   * @param title Test title.
+   * @param details See [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) for test details
+   * description.
+   * @param body Test body that takes one or two arguments: an object with fixtures and optional
+   * [TestInfo](https://playwright.dev/docs/api/class-testinfo).
+   * @param condition Test is marked as "should fail" when the condition is `true`.
+   * @param callback A function that returns whether to mark as "should fail", based on test fixtures. Test or tests are marked as
+   * "should fail" when the return value is `true`.
+   * @param description Optional description that will be reflected in a test report.
+   */
+  (): void;
+    /**
+   * You can use `test.fail.only` to focus on a specific test that is expected to fail. This is particularly useful when
+   * debugging a failing test or working on a specific issue.
+   *
+   * To declare a focused "failing" test:
+   * - `test.fail.only(title, body)`
+   * - `test.fail.only(title, details, body)`
+   *
+   * **Usage**
+   *
+   * You can declare a focused failing test, so that Playwright runs only this test and ensures it actually fails.
+   *
+   * ```js
+   * import { test, expect } from '@playwright/test';
+   *
+   * test.fail.only('focused failing test', async ({ page }) => {
+   *   // This test is expected to fail
+   * });
+   * test('not in the focused group', async ({ page }) => {
+   *   // This test will not run
+   * });
+   * ```
+   *
+   * @param title Test title.
+   * @param details See [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe) for test
+   * details description.
+   * @param body Test body that takes one or two arguments: an object with fixtures and optional
+   * [TestInfo](https://playwright.dev/docs/api/class-testinfo).
+   */
+  only: TestFunction<TestArgs & WorkerArgs>;
+  }
   /**
    * Marks a test as "slow". Slow test will be given triple the default timeout.
    *

tests/playwright-test/basic.spec.ts

@@ -153,6 +153,10 @@ test('should respect focused tests', async ({ runInlineTest }) => {
         });
       });
 
+      test.fail.only('focused fail.only test', () => {
+        expect(1 + 1).toBe(3);
+      });
+
       test.describe('non-focused describe', () => {
         test('describe test', () => {
           expect(1 + 1).toBe(3);
@@ -172,13 +176,46 @@ test('should respect focused tests', async ({ runInlineTest }) => {
         test.only('test4', () => {
           expect(1 + 1).toBe(2);
         });
+        test.fail.only('test5', () => {
+          expect(1 + 1).toBe(3);
+        });
       });
     `
   });
-  expect(passed).toBe(5);
+  expect(passed).toBe(7);
   expect(exitCode).toBe(0);
 });
 
+test('should respect focused tests with test.fail', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'fail-only.spec.ts': `
+      import { test, expect } from '@playwright/test';
+
+      test('test1', () => {
+        console.log('test1 should not run');
+        expect(1 + 1).toBe(2);
+      });
+
+      test.fail.only('test2', () => {
+        console.log('test2 should run and fail');
+        expect(1 + 1).toBe(3);
+      });
+
+      test('test3', () => {
+        console.log('test3 should not run');
+        expect(1 + 1).toBe(2);
+      });
+    `,
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+  expect(result.failed).toBe(0);
+  expect(result.skipped).toBe(0);
+  expect(result.output).toContain('test2 should run and fail');
+  expect(result.output).not.toContain('test1 should not run');
+  expect(result.output).not.toContain('test3 should not run');
+});
+
 test('skip should take priority over fail', async ({ runInlineTest }) => {
   const { passed, skipped, failed } = await runInlineTest({
     'test.spec.ts': `
@@ -550,3 +587,33 @@ test('should support describe.fixme', async ({ runInlineTest }) => {
   expect(result.skipped).toBe(3);
   expect(result.output).toContain('heytest4');
 });
+
+test('should fail when test.fail.only passes unexpectedly', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'fail-only-pass.spec.ts': `
+      import { test, expect } from '@playwright/test';
+
+      test('test1', () => {
+        console.log('test1 should not run');
+        expect(1 + 1).toBe(2);
+      });
+
+      test.fail.only('test2', () => {
+        console.log('test2 should run and pass unexpectedly');
+        expect(1 + 1).toBe(2);
+      });
+
+      test('test3', () => {
+        console.log('test3 should not run');
+        expect(1 + 1).toBe(2);
+      });
+    `,
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(0);
+  expect(result.failed).toBe(1);
+  expect(result.skipped).toBe(0);
+  expect(result.output).toContain('should run and pass unexpectedly');
+  expect(result.output).not.toContain('test1 should not run');
+  expect(result.output).not.toContain('test3 should not run');
+});

tests/playwright-test/test-modifiers.spec.ts

@@ -279,6 +279,33 @@ test.describe('test modifier annotations', () => {
     expectTest('focused fixme by suite', 'skipped', 'skipped', ['fixme']);
   });
 
+  test('should work with fail.only inside describe.only', async ({ runInlineTest }) => {
+    const result = await runInlineTest({
+      'a.test.ts': `
+        import { test, expect } from '@playwright/test';
+  
+        test.describe.only("suite", () => {
+          test.skip('focused skip by suite', () => {});
+          test.fixme('focused fixme by suite', () => {});
+          test.fail.only('focused fail by suite', () => { expect(1).toBe(2); });
+        });
+  
+        test.describe.skip('not focused', () => {
+          test('no marker', () => {});
+        });
+      `,
+    });
+    const expectTest = expectTestHelper(result);
+
+    expect(result.exitCode).toBe(0);
+    expect(result.passed).toBe(1);
+    expect(result.failed).toBe(0);
+    expect(result.skipped).toBe(0);
+    expectTest('focused skip by suite', 'skipped', 'skipped', ['skip']);
+    expectTest('focused fixme by suite', 'skipped', 'skipped', ['fixme']);
+    expectTest('focused fail by suite', 'failed', 'expected', ['fail']);
+  });
+
   test('should not multiple on retry', async ({ runInlineTest }) => {
     const result = await runInlineTest({
       'a.test.ts': `

tests/playwright-test/types-2.spec.ts

@@ -33,6 +33,7 @@ test('basics should work', async ({ runTSC }) => {
         test.skip('my test', async () => {});
         test.fixme('my test', async () => {});
         test.fail('my test', async () => {});
+        test.fail.only('my test', async () => {});
       });
       test.describe(() => {
         test('my test', () => {});
@@ -59,6 +60,7 @@ test('basics should work', async ({ runTSC }) => {
       test.fixme('title', { tag: '@foo' }, () => {});
       test.only('title', { tag: '@foo' }, () => {});
       test.fail('title', { tag: '@foo' }, () => {});
+      test.fail.only('title', { tag: '@foo' }, () => {});
       test.describe('title', { tag: '@foo' }, () => {});
       test.describe('title', { annotation: { type: 'issue' } }, () => {});
       // @ts-expect-error

utils/generate_types/overrides-test.d.ts

@@ -110,11 +110,14 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
   fixme(): void;
   fixme(condition: boolean, description?: string): void;
   fixme(callback: (args: TestArgs & WorkerArgs) => boolean, description?: string): void;
-  fail(title: string, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise<void> | void): void;
+  fail: {
-  fail(title: string, details: TestDetails, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise<void> | void): void;
+    (title: string, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise<void> | void): void;
-  fail(condition: boolean, description?: string): void;
+    (title: string, details: TestDetails, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise<void> | void): void;
-  fail(callback: (args: TestArgs & WorkerArgs) => boolean, description?: string): void;
+    (condition: boolean, description?: string): void;
-  fail(): void;
+    (callback: (args: TestArgs & WorkerArgs) => boolean, description?: string): void;
+    (): void;
+    only: TestFunction<TestArgs & WorkerArgs>;
+  }
   slow(): void;
   slow(condition: boolean, description?: string): void;
   slow(callback: (args: TestArgs & WorkerArgs) => boolean, description?: string): void;