diff --git a/docs/src/test-api/class-test.md b/docs/src/test-api/class-test.md index 31f60b7e9c..7ea05d4c56 100644 --- a/docs/src/test-api/class-test.md +++ b/docs/src/test-api/class-test.md @@ -1138,6 +1138,57 @@ Optional description that will be reflected in a test report. +## method: Test.fail.only +* since: v1.49 + +You can use `test.fail.only` to focus on a specific test that is expected to fail. This is particularly useful when debugging a failing test or working on a specific issue. + +To declare a focused "failing" test: +* `test.fail.only(title, body)` +* `test.fail.only(title, details, body)` + +**Usage** + +You can declare a focused failing test, so that Playwright runs only this test and ensures it actually fails. + +```js +import { test, expect } from '@playwright/test'; + +test.fail.only('focused failing test', async ({ page }) => { + // This test is expected to fail +}); +test('not in the focused group', async ({ page }) => { + // This test will not run +}); +``` + +### param: Test.fail.only.title +* since: v1.49 + +- `title` ?<[string]> + +Test title. + +### param: Test.fail.only.details +* since: v1.49 + +- `details` ?<[Object]> + - `tag` ?<[string]|[Array]<[string]>> + - `annotation` ?<[Object]|[Array]<[Object]>> + - `type` <[string]> + - `description` ?<[string]> + +See [`method: Test.describe`] for test details description. + +### param: Test.fail.only.body +* since: v1.49 + +- `body` ?<[function]\([Fixtures], [TestInfo]\)> + +Test body that takes one or two arguments: an object with fixtures and optional [TestInfo]. + + + ## method: Test.fixme * since: v1.10 diff --git a/packages/playwright/src/common/testType.ts b/packages/playwright/src/common/testType.ts index adf6bc3734..f22fd159d8 100644 --- a/packages/playwright/src/common/testType.ts +++ b/packages/playwright/src/common/testType.ts @@ -52,6 +52,7 @@ export class TestTypeImpl { test.skip = wrapFunctionWithLocation(this._modifier.bind(this, 'skip')); test.fixme = wrapFunctionWithLocation(this._modifier.bind(this, 'fixme')); test.fail = wrapFunctionWithLocation(this._modifier.bind(this, 'fail')); + test.fail.only = wrapFunctionWithLocation(this._createTest.bind(this, 'fail.only')); test.slow = wrapFunctionWithLocation(this._modifier.bind(this, 'slow')); test.setTimeout = wrapFunctionWithLocation(this._setTimeout.bind(this)); test.step = this._step.bind(this); @@ -81,7 +82,7 @@ export class TestTypeImpl { return suite; } - private _createTest(type: 'default' | 'only' | 'skip' | 'fixme' | 'fail', location: Location, title: string, fnOrDetails: Function | TestDetails, fn?: Function) { + private _createTest(type: 'default' | 'only' | 'skip' | 'fixme' | 'fail' | 'fail.only', location: Location, title: string, fnOrDetails: Function | TestDetails, fn?: Function) { throwIfRunningInsideJest(); const suite = this._currentSuite(location, 'test()'); if (!suite) @@ -104,10 +105,12 @@ export class TestTypeImpl { test._tags.push(...validatedDetails.tags); suite._addTest(test); - if (type === 'only') + if (type === 'only' || type === 'fail.only') test._only = true; if (type === 'skip' || type === 'fixme' || type === 'fail') test._staticAnnotations.push({ type }); + else if (type === 'fail.only') + test._staticAnnotations.push({ type: 'fail' }); } private _describe(type: 'default' | 'only' | 'serial' | 'serial.only' | 'parallel' | 'parallel.only' | 'skip' | 'fixme', location: Location, titleOrFn: string | Function, fnOrDetails?: TestDetails | Function, fn?: Function) { diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index c96de2091a..78f8eb7f4f 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -3625,8 +3625,8 @@ export interface TestType Promise | void): void; - /** + fail: { + /** * Marks a test as "should fail". Playwright runs this test and ensures that it is actually failing. This is useful * for documentation purposes to acknowledge that some functionality is broken until it is fixed. * @@ -3702,8 +3702,8 @@ export interface TestType Promise | void): void; - /** + (title: string, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | void): void; + /** * Marks a test as "should fail". Playwright runs this test and ensures that it is actually failing. This is useful * for documentation purposes to acknowledge that some functionality is broken until it is fixed. * @@ -3779,8 +3779,8 @@ export interface TestType Promise | void): void; + /** * Marks a test as "should fail". Playwright runs this test and ensures that it is actually failing. This is useful * for documentation purposes to acknowledge that some functionality is broken until it is fixed. * @@ -3856,8 +3856,8 @@ export interface TestType boolean, description?: string): void; - /** + (condition: boolean, description?: string): void; + /** * Marks a test as "should fail". Playwright runs this test and ensures that it is actually failing. This is useful * for documentation purposes to acknowledge that some functionality is broken until it is fixed. * @@ -3933,7 +3933,115 @@ export interface TestType boolean, description?: string): void; + /** + * Marks a test as "should fail". Playwright runs this test and ensures that it is actually failing. This is useful + * for documentation purposes to acknowledge that some functionality is broken until it is fixed. + * + * To declare a "failing" test: + * - `test.fail(title, body)` + * - `test.fail(title, details, body)` + * + * To annotate test as "failing" at runtime: + * - `test.fail(condition, description)` + * - `test.fail(callback, description)` + * - `test.fail()` + * + * **Usage** + * + * You can declare a test as failing, so that Playwright ensures it actually fails. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test.fail('not yet ready', async ({ page }) => { + * // ... + * }); + * ``` + * + * If your test fails in some configurations, but not all, you can mark the test as failing inside the test body based + * on some condition. We recommend passing a `description` argument in this case. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test('fail in WebKit', async ({ page, browserName }) => { + * test.fail(browserName === 'webkit', 'This feature is not implemented for Mac yet'); + * // ... + * }); + * ``` + * + * You can mark all tests in a file or + * [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe) group as + * "should fail" based on some condition with a single `test.fail(callback, description)` call. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test.fail(({ browserName }) => browserName === 'webkit', 'not implemented yet'); + * + * test('fail in WebKit 1', async ({ page }) => { + * // ... + * }); + * test('fail in WebKit 2', async ({ page }) => { + * // ... + * }); + * ``` + * + * You can also call `test.fail()` without arguments inside the test body to always mark the test as failed. We + * recommend declaring a failing test with `test.fail(title, body)` instead. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test('less readable', async ({ page }) => { + * test.fail(); + * // ... + * }); + * ``` + * + * @param title Test title. + * @param details See [test.(call)(title[, details, body])](https://playwright.dev/docs/api/class-test#test-call) for test details + * description. + * @param body Test body that takes one or two arguments: an object with fixtures and optional + * [TestInfo](https://playwright.dev/docs/api/class-testinfo). + * @param condition Test is marked as "should fail" when the condition is `true`. + * @param callback A function that returns whether to mark as "should fail", based on test fixtures. Test or tests are marked as + * "should fail" when the return value is `true`. + * @param description Optional description that will be reflected in a test report. + */ + (): void; + /** + * You can use `test.fail.only` to focus on a specific test that is expected to fail. This is particularly useful when + * debugging a failing test or working on a specific issue. + * + * To declare a focused "failing" test: + * - `test.fail.only(title, body)` + * - `test.fail.only(title, details, body)` + * + * **Usage** + * + * You can declare a focused failing test, so that Playwright runs only this test and ensures it actually fails. + * + * ```js + * import { test, expect } from '@playwright/test'; + * + * test.fail.only('focused failing test', async ({ page }) => { + * // This test is expected to fail + * }); + * test('not in the focused group', async ({ page }) => { + * // This test will not run + * }); + * ``` + * + * @param title Test title. + * @param details See [test.describe([title, details, callback])](https://playwright.dev/docs/api/class-test#test-describe) for test + * details description. + * @param body Test body that takes one or two arguments: an object with fixtures and optional + * [TestInfo](https://playwright.dev/docs/api/class-testinfo). + */ + only: TestFunction; + } /** * Marks a test as "slow". Slow test will be given triple the default timeout. * diff --git a/tests/playwright-test/basic.spec.ts b/tests/playwright-test/basic.spec.ts index 3b47603c25..ce6825c559 100644 --- a/tests/playwright-test/basic.spec.ts +++ b/tests/playwright-test/basic.spec.ts @@ -153,6 +153,10 @@ test('should respect focused tests', async ({ runInlineTest }) => { }); }); + test.fail.only('focused fail.only test', () => { + expect(1 + 1).toBe(3); + }); + test.describe('non-focused describe', () => { test('describe test', () => { expect(1 + 1).toBe(3); @@ -172,13 +176,46 @@ test('should respect focused tests', async ({ runInlineTest }) => { test.only('test4', () => { expect(1 + 1).toBe(2); }); + test.fail.only('test5', () => { + expect(1 + 1).toBe(3); + }); }); ` }); - expect(passed).toBe(5); + expect(passed).toBe(7); expect(exitCode).toBe(0); }); +test('should respect focused tests with test.fail', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'fail-only.spec.ts': ` + import { test, expect } from '@playwright/test'; + + test('test1', () => { + console.log('test1 should not run'); + expect(1 + 1).toBe(2); + }); + + test.fail.only('test2', () => { + console.log('test2 should run and fail'); + expect(1 + 1).toBe(3); + }); + + test('test3', () => { + console.log('test3 should not run'); + expect(1 + 1).toBe(2); + }); + `, + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.failed).toBe(0); + expect(result.skipped).toBe(0); + expect(result.output).toContain('test2 should run and fail'); + expect(result.output).not.toContain('test1 should not run'); + expect(result.output).not.toContain('test3 should not run'); +}); + test('skip should take priority over fail', async ({ runInlineTest }) => { const { passed, skipped, failed } = await runInlineTest({ 'test.spec.ts': ` @@ -550,3 +587,33 @@ test('should support describe.fixme', async ({ runInlineTest }) => { expect(result.skipped).toBe(3); expect(result.output).toContain('heytest4'); }); + +test('should fail when test.fail.only passes unexpectedly', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'fail-only-pass.spec.ts': ` + import { test, expect } from '@playwright/test'; + + test('test1', () => { + console.log('test1 should not run'); + expect(1 + 1).toBe(2); + }); + + test.fail.only('test2', () => { + console.log('test2 should run and pass unexpectedly'); + expect(1 + 1).toBe(2); + }); + + test('test3', () => { + console.log('test3 should not run'); + expect(1 + 1).toBe(2); + }); + `, + }); + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(0); + expect(result.failed).toBe(1); + expect(result.skipped).toBe(0); + expect(result.output).toContain('should run and pass unexpectedly'); + expect(result.output).not.toContain('test1 should not run'); + expect(result.output).not.toContain('test3 should not run'); +}); diff --git a/tests/playwright-test/test-modifiers.spec.ts b/tests/playwright-test/test-modifiers.spec.ts index 0dd41bd0ab..f7fb9a6ae0 100644 --- a/tests/playwright-test/test-modifiers.spec.ts +++ b/tests/playwright-test/test-modifiers.spec.ts @@ -279,6 +279,33 @@ test.describe('test modifier annotations', () => { expectTest('focused fixme by suite', 'skipped', 'skipped', ['fixme']); }); + test('should work with fail.only inside describe.only', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + + test.describe.only("suite", () => { + test.skip('focused skip by suite', () => {}); + test.fixme('focused fixme by suite', () => {}); + test.fail.only('focused fail by suite', () => { expect(1).toBe(2); }); + }); + + test.describe.skip('not focused', () => { + test('no marker', () => {}); + }); + `, + }); + const expectTest = expectTestHelper(result); + + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(result.failed).toBe(0); + expect(result.skipped).toBe(0); + expectTest('focused skip by suite', 'skipped', 'skipped', ['skip']); + expectTest('focused fixme by suite', 'skipped', 'skipped', ['fixme']); + expectTest('focused fail by suite', 'failed', 'expected', ['fail']); + }); + test('should not multiple on retry', async ({ runInlineTest }) => { const result = await runInlineTest({ 'a.test.ts': ` diff --git a/tests/playwright-test/types-2.spec.ts b/tests/playwright-test/types-2.spec.ts index e61d4870ed..f794e06798 100644 --- a/tests/playwright-test/types-2.spec.ts +++ b/tests/playwright-test/types-2.spec.ts @@ -33,6 +33,7 @@ test('basics should work', async ({ runTSC }) => { test.skip('my test', async () => {}); test.fixme('my test', async () => {}); test.fail('my test', async () => {}); + test.fail.only('my test', async () => {}); }); test.describe(() => { test('my test', () => {}); @@ -59,6 +60,7 @@ test('basics should work', async ({ runTSC }) => { test.fixme('title', { tag: '@foo' }, () => {}); test.only('title', { tag: '@foo' }, () => {}); test.fail('title', { tag: '@foo' }, () => {}); + test.fail.only('title', { tag: '@foo' }, () => {}); test.describe('title', { tag: '@foo' }, () => {}); test.describe('title', { annotation: { type: 'issue' } }, () => {}); // @ts-expect-error diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index be1fa7ee37..54fffb5345 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -110,11 +110,14 @@ export interface TestType boolean, description?: string): void; - fail(title: string, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | void): void; - fail(title: string, details: TestDetails, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | void): void; - fail(condition: boolean, description?: string): void; - fail(callback: (args: TestArgs & WorkerArgs) => boolean, description?: string): void; - fail(): void; + fail: { + (title: string, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | void): void; + (title: string, details: TestDetails, body: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | void): void; + (condition: boolean, description?: string): void; + (callback: (args: TestArgs & WorkerArgs) => boolean, description?: string): void; + (): void; + only: TestFunction; + } slow(): void; slow(condition: boolean, description?: string): void; slow(callback: (args: TestArgs & WorkerArgs) => boolean, description?: string): void;