chore: add page snapshot on test end (#34573)

Signed-off-by: Simon Knott <info@simonknott.de>
Co-authored-by: Yury Semikhatsky <yurys@chromium.org>

packages/playwright/src/index.ts

@@ -54,6 +54,7 @@ type WorkerFixtures = PlaywrightWorkerArgs & PlaywrightWorkerOptions & {
   _optionContextReuseMode: ContextReuseMode,
   _optionConnectOptions: PlaywrightWorkerOptions['connectOptions'],
   _reuseContext: boolean,
+  _pageSnapshot: PageSnapshotOption,
 };
 
 const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
@@ -71,6 +72,7 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
   screenshot: ['off', { scope: 'worker', option: true }],
   video: ['off', { scope: 'worker', option: true }],
   trace: ['off', { scope: 'worker', option: true }],
+  _pageSnapshot: ['off', { scope: 'worker', option: true }],
 
   _browserOptions: [async ({ playwright, headless, channel, launchOptions }, use) => {
     const options: LaunchOptions = {
@@ -247,13 +249,13 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
     }
   }, { auto: 'all-hooks-included',  title: 'context configuration', box: true } as any],
 
-  _setupArtifacts: [async ({ playwright, screenshot }, use, testInfo) => {
+  _setupArtifacts: [async ({ playwright, screenshot, _pageSnapshot }, use, testInfo) => {
     // This fixture has a separate zero-timeout slot to ensure that artifact collection
     // happens even after some fixtures or hooks time out.
     // Now that default test timeout is known, we can replace zero with an actual value.
     testInfo.setTimeout(testInfo.project.timeout);
 
-    const artifactsRecorder = new ArtifactsRecorder(playwright, tracing().artifactsDir(), screenshot);
+    const artifactsRecorder = new ArtifactsRecorder(playwright, tracing().artifactsDir(), screenshot, _pageSnapshot);
     await artifactsRecorder.willStartTest(testInfo as TestInfoImpl);
 
     const tracingGroupSteps: TestStepInternal[] = [];
@@ -452,6 +454,7 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
 
 type ScreenshotOption = PlaywrightWorkerOptions['screenshot'] | undefined;
 type Playwright = PlaywrightWorkerArgs['playwright'];
+type PageSnapshotOption = 'off' | 'on' | 'only-on-failure';
 
 function normalizeVideoMode(video: VideoMode | 'retry-with-video' | { mode: VideoMode } | undefined): VideoMode {
   if (!video)
@@ -523,41 +526,132 @@ function connectOptionsFromEnv() {
   };
 }
 
-class ArtifactsRecorder {
+class SnapshotRecorder {
-  private _testInfo!: TestInfoImpl;
+  private _ordinal = 0;
-  private _playwright: Playwright;
+  private _temporary: string[] = [];
-  private _artifactsDir: string;
+  private _snapshottedSymbol = Symbol('snapshotted');
-  private _screenshotMode: ScreenshotMode;
-  private _screenshotOptions: { mode: ScreenshotMode } & Pick<playwrightLibrary.PageScreenshotOptions, 'fullPage' | 'omitBackground'> | undefined;
-  private _temporaryScreenshots: string[] = [];
-  private _temporaryArtifacts: string[] = [];
-  private _reusedContexts = new Set<BrowserContext>();
-  private _screenshotOrdinal = 0;
-  private _screenshottedSymbol: symbol;
-  private _startedCollectingArtifacts: symbol;
 
-  constructor(playwright: Playwright, artifactsDir: string, screenshot: ScreenshotOption) {
+  constructor(
-    this._playwright = playwright;
+    private _artifactsRecorder: ArtifactsRecorder,
-    this._artifactsDir = artifactsDir;
+    private _mode: ScreenshotMode | PageSnapshotOption,
-    this._screenshotMode = normalizeScreenshotMode(screenshot);
+    private _name: string,
-    this._screenshotOptions = typeof screenshot === 'string' ? undefined : screenshot;
+    private _contentType: string,
-    this._screenshottedSymbol = Symbol('screenshotted');
+    private _extension: string,
-    this._startedCollectingArtifacts = Symbol('startedCollectingArtifacts');
+    private _doSnapshot: (page: Page, path: string) => Promise<void>) {
+  }
+
+  fixOrdinal() {
+    // Since beforeAll(s), test and afterAll(s) reuse the same TestInfo, make sure we do not
+    // overwrite previous screenshots.
+    this._ordinal = this.testInfo.attachments.filter(a => a.name === this._name).length;
+  }
+
+  private shouldCaptureUponFinish() {
+    return this._mode === 'on' ||
+        (this._mode === 'only-on-failure' && this.testInfo._isFailure()) ||
+        (this._mode === 'on-first-failure' && this.testInfo._isFailure() && this.testInfo.retry === 0);
+  }
+
+  async maybeCapture() {
+    if (!this.shouldCaptureUponFinish())
+      return;
+
+    const contexts: BrowserContext[] = [];
+    const playwright = this._artifactsRecorder._playwright;
+    for (const browserType of [playwright.chromium, playwright.firefox, playwright.webkit])
+      contexts.push(...(browserType as any)._contexts);
+
+    await Promise.all(contexts.flatMap(context => context.pages().map(page => this._snapshotPage(page, false))));
+  }
+
+  async persistTemporary() {
+    if (this.shouldCaptureUponFinish()) {
+      await Promise.all(this._temporary.map(async file => {
+        try {
+          const path = this._createAttachmentPath();
+          await fs.promises.rename(file, path);
+          this._attach(path);
+        } catch {
+        }
+      }));
+    }
+  }
+
+  async captureTemporary(context: BrowserContext) {
+    if (this._mode === 'on' || this._mode === 'only-on-failure' || (this._mode === 'on-first-failure' && this.testInfo.retry === 0))
+      await Promise.all(context.pages().map(page => this._snapshotPage(page, true)));
+  }
+
+  private _attach(screenshotPath: string) {
+    this.testInfo.attachments.push({ name: this._name, path: screenshotPath, contentType: this._contentType });
+  }
+
+  private _createAttachmentPath() {
+    const testFailed = this.testInfo._isFailure();
+    const index = this._ordinal + 1;
+    ++this._ordinal;
+    const path = this.testInfo.outputPath(`test-${testFailed ? 'failed' : 'finished'}-${index}${this._extension}`);
+    return path;
   }
 
   private _createTemporaryArtifact(...name: string[]) {
-    const file = path.join(this._artifactsDir, ...name);
+    const file = path.join(this._artifactsRecorder._artifactsDir, ...name);
-    this._temporaryArtifacts.push(file);
     return file;
   }
 
+  private async _snapshotPage(page: Page, temporary: boolean) {
+    if ((page as any)[this._snapshottedSymbol])
+      return;
+    (page as any)[this._snapshottedSymbol] = true;
+    try {
+      const path = temporary ? this._createTemporaryArtifact(createGuid() + this._extension) : this._createAttachmentPath();
+      await this._doSnapshot(page, path);
+      if (temporary)
+        this._temporary.push(path);
+      else
+        this._attach(path);
+    } catch {
+      // snapshot may fail, just ignore.
+    }
+  }
+
+  private get testInfo(): TestInfoImpl {
+    return this._artifactsRecorder._testInfo;
+  }
+}
+
+class ArtifactsRecorder {
+  _testInfo!: TestInfoImpl;
+  _playwright: Playwright;
+  _artifactsDir: string;
+  private _reusedContexts = new Set<BrowserContext>();
+  private _startedCollectingArtifacts: symbol;
+
+  private _pageSnapshotRecorder: SnapshotRecorder;
+  private _screenshotRecorder: SnapshotRecorder;
+
+  constructor(playwright: Playwright, artifactsDir: string, screenshot: ScreenshotOption, pageSnapshot: PageSnapshotOption) {
+    this._playwright = playwright;
+    this._artifactsDir = artifactsDir;
+    const screenshotOptions = typeof screenshot === 'string' ? undefined : screenshot;
+    this._startedCollectingArtifacts = Symbol('startedCollectingArtifacts');
+
+    this._screenshotRecorder = new SnapshotRecorder(this, normalizeScreenshotMode(screenshot), 'screenshot', 'image/png', '.png', async (page, path) => {
+      await page.screenshot({ ...screenshotOptions, timeout: 5000, path, caret: 'initial' });
+    });
+
+    this._pageSnapshotRecorder = new SnapshotRecorder(this, pageSnapshot, 'pageSnapshot', 'text/plain', '.ariasnapshot', async (page, path) => {
+      const ariaSnapshot = await page.locator('body').ariaSnapshot();
+      await fs.promises.writeFile(path, ariaSnapshot);
+    });
+  }
+
   async willStartTest(testInfo: TestInfoImpl) {
     this._testInfo = testInfo;
     testInfo._onDidFinishTestFunction = () => this.didFinishTestFunction();
 
-    // Since beforeAll(s), test and afterAll(s) reuse the same TestInfo, make sure we do not
+    this._screenshotRecorder.fixOrdinal();
-    // overwrite previous screenshots.
+    this._pageSnapshotRecorder.fixOrdinal();
-    this._screenshotOrdinal = testInfo.attachments.filter(a => a.name === 'screenshot').length;
 
     // Process existing contexts.
     for (const browserType of [this._playwright.chromium, this._playwright.firefox, this._playwright.webkit]) {
@@ -587,11 +681,9 @@ class ArtifactsRecorder {
     if (this._reusedContexts.has(context))
       return;
     await this._stopTracing(context.tracing);
-    if (this._screenshotMode === 'on' || this._screenshotMode === 'only-on-failure' || (this._screenshotMode === 'on-first-failure' && this._testInfo.retry === 0)) {
+
-      // Capture screenshot for now. We'll know whether we have to preserve them
+    await this._screenshotRecorder.captureTemporary(context);
-      // after the test finishes.
+    await this._pageSnapshotRecorder.captureTemporary(context);
-      await Promise.all(context.pages().map(page => this._screenshotPage(page, true)));
-    }
   }
 
   async didCreateRequestContext(context: APIRequestContext) {
@@ -604,21 +696,13 @@ class ArtifactsRecorder {
     await this._stopTracing(tracing);
   }
 
-  private _shouldCaptureScreenshotUponFinish() {
-    return this._screenshotMode === 'on' ||
-        (this._screenshotMode === 'only-on-failure' && this._testInfo._isFailure()) ||
-        (this._screenshotMode === 'on-first-failure' && this._testInfo._isFailure() && this._testInfo.retry === 0);
-  }
-
   async didFinishTestFunction() {
-    if (this._shouldCaptureScreenshotUponFinish())
+    await this._screenshotRecorder.maybeCapture();
-      await this._screenshotOnTestFailure();
+    await this._pageSnapshotRecorder.maybeCapture();
   }
 
   async didFinishTest() {
-    const captureScreenshots = this._shouldCaptureScreenshotUponFinish();
+    await this.didFinishTestFunction();
-    if (captureScreenshots)
-      await this._screenshotOnTestFailure();
 
     let leftoverContexts: BrowserContext[] = [];
     for (const browserType of [this._playwright.chromium, this._playwright.firefox, this._playwright.webkit])
@@ -634,55 +718,8 @@ class ArtifactsRecorder {
       await this._stopTracing(tracing);
     })));
 
-    // Attach temporary screenshots for contexts closed before collecting the test trace.
+    await this._screenshotRecorder.persistTemporary();
-    if (captureScreenshots) {
+    await this._pageSnapshotRecorder.persistTemporary();
-      for (const file of this._temporaryScreenshots) {
-        try {
-          const screenshotPath = this._createScreenshotAttachmentPath();
-          await fs.promises.rename(file, screenshotPath);
-          this._attachScreenshot(screenshotPath);
-        } catch {
-        }
-      }
-    }
-  }
-
-  private _createScreenshotAttachmentPath() {
-    const testFailed = this._testInfo._isFailure();
-    const index = this._screenshotOrdinal + 1;
-    ++this._screenshotOrdinal;
-    const screenshotPath = this._testInfo.outputPath(`test-${testFailed ? 'failed' : 'finished'}-${index}.png`);
-    return screenshotPath;
-  }
-
-  private async _screenshotPage(page: Page, temporary: boolean) {
-    if ((page as any)[this._screenshottedSymbol])
-      return;
-    (page as any)[this._screenshottedSymbol] = true;
-    try {
-      const screenshotPath = temporary ? this._createTemporaryArtifact(createGuid() + '.png') : this._createScreenshotAttachmentPath();
-      // Pass caret=initial to avoid any evaluations that might slow down the screenshot
-      // and let the page modify itself from the problematic state it had at the moment of failure.
-      await page.screenshot({ ...this._screenshotOptions, timeout: 5000, path: screenshotPath, caret: 'initial' });
-      if (temporary)
-        this._temporaryScreenshots.push(screenshotPath);
-      else
-        this._attachScreenshot(screenshotPath);
-    } catch {
-      // Screenshot may fail, just ignore.
-    }
-  }
-
-  private _attachScreenshot(screenshotPath: string) {
-    this._testInfo.attachments.push({ name: 'screenshot', path: screenshotPath, contentType: 'image/png' });
-  }
-
-  private async _screenshotOnTestFailure() {
-    const contexts: BrowserContext[] = [];
-    for (const browserType of [this._playwright.chromium, this._playwright.firefox, this._playwright.webkit])
-      contexts.push(...(browserType as any)._contexts);
-    const pages = contexts.map(ctx => ctx.pages()).flat();
-    await Promise.all(pages.map(page => this._screenshotPage(page, false)));
   }
 
   private async _startTraceChunkOnContextCreation(tracing: Tracing) {

packages/playwright/src/isomorphic/testServerInterface.ts

@@ -96,6 +96,7 @@ export interface TestServerInterface {
     workers?: number | string;
     updateSnapshots?: 'all' | 'changed' | 'missing' | 'none';
     updateSourceMethod?: 'overwrite' | 'patch' | '3way';
+    pageSnapshot?: 'off' | 'on' | 'only-on-failure';
     reporters?: string[],
     trace?: 'on' | 'off';
     video?: 'on' | 'off';

packages/playwright/src/runner/testServer.ts

@@ -311,6 +311,7 @@ export class TestServerDispatcher implements TestServerInterface {
         ...(params.headed !== undefined ? { headless: !params.headed } : {}),
         _optionContextReuseMode: params.reuseContext ? 'when-possible' : undefined,
         _optionConnectOptions: params.connectWsEndpoint ? { wsEndpoint: params.connectWsEndpoint } : undefined,
+        _pageSnapshot: params.pageSnapshot,
       },
       ...(params.updateSnapshots ? { updateSnapshots: params.updateSnapshots } : {}),
       ...(params.updateSourceMethod ? { updateSourceMethod: params.updateSourceMethod } : {}),

tests/playwright-test/playwright.artifacts.spec.ts

@@ -420,3 +420,71 @@ test('should take screenshot when page is closed in afterEach', async ({ runInli
   expect(result.failed).toBe(1);
   expect(fs.existsSync(testInfo.outputPath('test-results', 'a-fails', 'test-failed-1.png'))).toBeTruthy();
 });
+
+test('should work with _pageSnapshot: on', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    ...testFiles,
+    'playwright.config.ts': `
+      module.exports = { use: { _pageSnapshot: 'on' } };
+    `,
+  }, { workers: 1 });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(5);
+  expect(result.failed).toBe(5);
+  expect(listFiles(testInfo.outputPath('test-results'))).toEqual([
+    '.last-run.json',
+    'artifacts-failing',
+    '  test-failed-1.ariasnapshot',
+    'artifacts-own-context-failing',
+    '  test-failed-1.ariasnapshot',
+    'artifacts-own-context-passing',
+    '  test-finished-1.ariasnapshot',
+    'artifacts-passing',
+    '  test-finished-1.ariasnapshot',
+    'artifacts-persistent-failing',
+    '  test-failed-1.ariasnapshot',
+    'artifacts-persistent-passing',
+    '  test-finished-1.ariasnapshot',
+    'artifacts-shared-shared-failing',
+    '  test-failed-1.ariasnapshot',
+    '  test-failed-2.ariasnapshot',
+    'artifacts-shared-shared-passing',
+    '  test-finished-1.ariasnapshot',
+    '  test-finished-2.ariasnapshot',
+    'artifacts-two-contexts',
+    '  test-finished-1.ariasnapshot',
+    '  test-finished-2.ariasnapshot',
+    'artifacts-two-contexts-failing',
+    '  test-failed-1.ariasnapshot',
+    '  test-failed-2.ariasnapshot',
+  ]);
+});
+
+test('should work with _pageSnapshot: only-on-failure', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    ...testFiles,
+    'playwright.config.ts': `
+      module.exports = { use: { _pageSnapshot: 'only-on-failure' } };
+    `,
+  }, { workers: 1 });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(5);
+  expect(result.failed).toBe(5);
+  expect(listFiles(testInfo.outputPath('test-results'))).toEqual([
+    '.last-run.json',
+    'artifacts-failing',
+    '  test-failed-1.ariasnapshot',
+    'artifacts-own-context-failing',
+    '  test-failed-1.ariasnapshot',
+    'artifacts-persistent-failing',
+    '  test-failed-1.ariasnapshot',
+    'artifacts-shared-shared-failing',
+    '  test-failed-1.ariasnapshot',
+    '  test-failed-2.ariasnapshot',
+    'artifacts-two-contexts-failing',
+    '  test-failed-1.ariasnapshot',
+    '  test-failed-2.ariasnapshot',
+  ]);
+});