From 2e01154bb57f5cb34c805092e0119742c46b95d8 Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Thu, 24 Oct 2024 04:41:35 -0700 Subject: [PATCH] feat: screenshot:on-first-failure (#33266) --- docs/src/test-api/class-testoptions.md | 5 ++-- packages/playwright/src/index.ts | 13 ++++++--- packages/playwright/types/test.d.ts | 3 ++- .../playwright.artifacts.spec.ts | 27 +++++++++++++++++++ utils/generate_types/overrides-test.d.ts | 2 +- 5 files changed, 42 insertions(+), 8 deletions(-) diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.md index 47bfd1f377..20dad210e7 100644 --- a/docs/src/test-api/class-testoptions.md +++ b/docs/src/test-api/class-testoptions.md @@ -479,8 +479,8 @@ export default defineConfig({ ## property: TestOptions.screenshot * since: v1.10 -- type: <[Object]|[ScreenshotMode]<"off"|"on"|"only-on-failure">> - - `mode` <[ScreenshotMode]<"off"|"on"|"only-on-failure">> Automatic screenshot mode. +- type: <[Object]|[ScreenshotMode]<"off"|"on"|"only-on-failure"|"on-first-failure">> + - `mode` <[ScreenshotMode]<"off"|"on"|"only-on-failure"|"on-first-failure">> Automatic screenshot mode. - `fullPage` ?<[boolean]> When true, takes a screenshot of the full scrollable page, instead of the currently visible viewport. Defaults to `false`. - `omitBackground` ?<[boolean]> Hides default white background and allows capturing screenshots with transparency. Not applicable to `jpeg` images. Defaults to `false`. @@ -488,6 +488,7 @@ Whether to automatically capture a screenshot after each test. Defaults to `'off * `'off'`: Do not capture screenshots. * `'on'`: Capture screenshot after each test. * `'only-on-failure'`: Capture screenshot after each test failure. +* `'on-first-failure'`: Capture screenshot after each test's first failure. **Usage** diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts index add1661502..c2c9596b6e 100644 --- a/packages/playwright/src/index.ts +++ b/packages/playwright/src/index.ts @@ -571,7 +571,7 @@ class ArtifactsRecorder { if (this._reusedContexts.has(context)) return; await this._stopTracing(context.tracing); - if (this._screenshotMode === 'on' || this._screenshotMode === 'only-on-failure') { + if (this._screenshotMode === 'on' || this._screenshotMode === 'only-on-failure' || (this._screenshotMode === 'on-first-failure' && this._testInfo.retry === 0)) { // Capture screenshot for now. We'll know whether we have to preserve them // after the test finishes. await Promise.all(context.pages().map(page => this._screenshotPage(page, true))); @@ -588,14 +588,19 @@ class ArtifactsRecorder { await this._stopTracing(tracing); } + private _shouldCaptureScreenshotUponFinish() { + return this._screenshotMode === 'on' || + (this._screenshotMode === 'only-on-failure' && this._testInfo._isFailure()) || + (this._screenshotMode === 'on-first-failure' && this._testInfo._isFailure() && this._testInfo.retry === 0); + } + async didFinishTestFunction() { - const captureScreenshots = this._screenshotMode === 'on' || (this._screenshotMode === 'only-on-failure' && this._testInfo._isFailure()); - if (captureScreenshots) + if (this._shouldCaptureScreenshotUponFinish()) await this._screenshotOnTestFailure(); } async didFinishTest() { - const captureScreenshots = this._screenshotMode === 'on' || (this._screenshotMode === 'only-on-failure' && this._testInfo._isFailure()); + const captureScreenshots = this._shouldCaptureScreenshotUponFinish(); if (captureScreenshots) await this._screenshotOnTestFailure(); diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index ae02d1506e..5db30f72e2 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -5863,6 +5863,7 @@ export interface PlaywrightWorkerOptions { * - `'off'`: Do not capture screenshots. * - `'on'`: Capture screenshot after each test. * - `'only-on-failure'`: Capture screenshot after each test failure. + * - `'on-first-failure'`: Capture screenshot after each test's first failure. * * **Usage** * @@ -5938,7 +5939,7 @@ export interface PlaywrightWorkerOptions { video: VideoMode | /** deprecated */ 'retry-with-video' | { mode: VideoMode, size?: ViewportSize }; } -export type ScreenshotMode = 'off' | 'on' | 'only-on-failure'; +export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failure'; export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure'; export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry'; diff --git a/tests/playwright-test/playwright.artifacts.spec.ts b/tests/playwright-test/playwright.artifacts.spec.ts index b666aa4b70..2e3d99766b 100644 --- a/tests/playwright-test/playwright.artifacts.spec.ts +++ b/tests/playwright-test/playwright.artifacts.spec.ts @@ -192,6 +192,33 @@ test('should work with screenshot: only-on-failure', async ({ runInlineTest }, t ]); }); +test('should work with screenshot: on-first-failure', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', async ({ page }) => { + await page.setContent('I am the page'); + expect(1).toBe(2); + }); + `, + 'playwright.config.ts': ` + module.exports = { + retries: 1, + use: { screenshot: 'on-first-failure' } + }; + `, + }, { workers: 1 }); + + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(0); + expect(result.failed).toBe(1); + expect(listFiles(testInfo.outputPath('test-results'))).toEqual([ + '.last-run.json', + 'a-fails', + ' test-failed-1.png', + ]); +}); + test('should work with screenshot: only-on-failure & fullPage', async ({ runInlineTest, server }, testInfo) => { const result = await runInlineTest({ 'artifacts.spec.ts': ` diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index ff46ba0e5c..49a7093dd3 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -236,7 +236,7 @@ export interface PlaywrightWorkerOptions { video: VideoMode | /** deprecated */ 'retry-with-video' | { mode: VideoMode, size?: ViewportSize }; } -export type ScreenshotMode = 'off' | 'on' | 'only-on-failure'; +export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failure'; export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure'; export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';