feat(tracing): clip `canvas` contents from screenshots (#33119)

packages/playwright-core/src/server/trace/test/inMemorySnapshotter.ts

@@ -85,7 +85,7 @@ export class InMemorySnapshotter implements SnapshotterDelegate, HarTracerDelega
 
   onFrameSnapshot(snapshot: FrameSnapshot): void {
     ++this._snapshotCount;
-    const renderer = this._storage.addFrameSnapshot(snapshot);
+    const renderer = this._storage.addFrameSnapshot(snapshot, []);
     this._snapshotReadyPromises.get(snapshot.snapshotName || '')?.resolve(renderer);
   }
 

packages/trace-viewer/src/sw/main.ts

@@ -129,6 +129,13 @@ async function doFetch(event: FetchEvent): Promise<Response> {
       return response;
     }
 
+    if (relativePath.startsWith('/closest-screenshot/')) {
+      const { snapshotServer } = loadedTraces.get(traceUrl!) || {};
+      if (!snapshotServer)
+        return new Response(null, { status: 404 });
+      return snapshotServer.serveClosestScreenshot(relativePath, url.searchParams);
+    }
+
     if (relativePath.startsWith('/sha1/')) {
       // Sha1 for sources is based on the file path, can't load it of a random model.
       const sha1 = relativePath.slice('/sha1/'.length);

packages/trace-viewer/src/sw/snapshotRenderer.ts

@@ -16,6 +16,16 @@
 
 import { escapeHTMLAttribute, escapeHTML } from '@isomorphic/stringUtils';
 import type { FrameSnapshot, NodeNameAttributesChildNodesSnapshot, NodeSnapshot, RenderedFrameSnapshot, ResourceSnapshot, SubtreeReferenceSnapshot } from '@trace/snapshot';
+import type { PageEntry } from '../types/entries';
+
+function findClosest<T>(items: T[], metric: (v: T) => number, target: number) {
+  return items.find((item, index) => {
+    if (index === items.length - 1)
+      return true;
+    const next = items[index + 1];
+    return Math.abs(metric(item) - target) < Math.abs(metric(next) - target);
+  });
+}
 
 function isNodeNameAttributesChildNodesSnapshot(n: NodeSnapshot): n is NodeNameAttributesChildNodesSnapshot {
   return Array.isArray(n) && typeof n[0] === 'string';
@@ -60,13 +70,15 @@ export class SnapshotRenderer {
   private _resources: ResourceSnapshot[];
   private _snapshot: FrameSnapshot;
   private _callId: string;
+  private _screencastFrames: PageEntry['screencastFrames'];
 
-  constructor(resources: ResourceSnapshot[], snapshots: FrameSnapshot[], index: number) {
+  constructor(resources: ResourceSnapshot[], snapshots: FrameSnapshot[], screencastFrames: PageEntry['screencastFrames'], index: number) {
     this._resources = resources;
     this._snapshots = snapshots;
     this._index = index;
     this._snapshot = snapshots[index];
     this._callId = snapshots[index].callId;
+    this._screencastFrames = screencastFrames;
     this.snapshotName = snapshots[index].snapshotName;
   }
 
@@ -78,6 +90,14 @@ export class SnapshotRenderer {
     return this._snapshots[this._index].viewport;
   }
 
+  closestScreenshot(): string | undefined {
+    const { wallTime, timestamp } = this.snapshot();
+    const closestFrame = (wallTime && this._screencastFrames[0]?.frameSwapWallTime)
+      ? findClosest(this._screencastFrames, frame => frame.frameSwapWallTime!, wallTime)
+      : findClosest(this._screencastFrames, frame => frame.timestamp, timestamp);
+    return closestFrame?.sha1;
+  }
+
   render(): RenderedFrameSnapshot {
     const result: string[] = [];
     const visit = (n: NodeSnapshot, snapshotIndex: number, parentTag: string | undefined, parentAttrs: [string, string][] | undefined) => {
@@ -244,6 +264,8 @@ function snapshotNodes(snapshot: FrameSnapshot): NodeSnapshot[] {
 
 function snapshotScript(...targetIds: (string | undefined)[]) {
   function applyPlaywrightAttributes(unwrapPopoutUrl: (url: string) => string, ...targetIds: (string | undefined)[]) {
+    const isUnderTest = new URLSearchParams(location.search).has('isUnderTest');
+
     const kPointerWarningTitle = 'Recorded click position in absolute coordinates did not' +
         ' match the center of the clicked element. This is likely due to a difference between' +
         ' the test runner and the trace viewer operating systems.';
@@ -251,6 +273,7 @@ function snapshotScript(...targetIds: (string | undefined)[]) {
     const scrollTops: Element[] = [];
     const scrollLefts: Element[] = [];
     const targetElements: Element[] = [];
+    const canvasElements: HTMLCanvasElement[] = [];
 
     const visit = (root: Document | ShadowRoot) => {
       // Collect all scrolled elements for later use.
@@ -326,6 +349,8 @@ function snapshotScript(...targetIds: (string | undefined)[]) {
         }
         (root as any).adoptedStyleSheets = adoptedSheets;
       }
+
+      canvasElements.push(...root.querySelectorAll('canvas'));
     };
 
     const onLoad = () => {
@@ -342,12 +367,12 @@ function snapshotScript(...targetIds: (string | undefined)[]) {
       document.styleSheets[0].disabled = true;
 
       const search = new URL(window.location.href).searchParams;
+      const isTopFrame = window.location.pathname.match(/\/page@[a-z0-9]+$/);
 
       if (search.get('pointX') && search.get('pointY')) {
         const pointX = +search.get('pointX')!;
         const pointY = +search.get('pointY')!;
         const hasInputTarget = search.has('hasInputTarget');
-        const isTopFrame = window.location.pathname.match(/\/page@[a-z0-9]+$/);
         const hasTargetElements = targetElements.length > 0;
         const roots = document.documentElement ? [document.documentElement] : [];
         for (const target of (hasTargetElements ? targetElements : roots)) {
@@ -393,6 +418,76 @@ function snapshotScript(...targetIds: (string | undefined)[]) {
           }
         }
       }
+
+      if (canvasElements.length > 0) {
+        function drawCheckerboard(context: CanvasRenderingContext2D, canvas: HTMLCanvasElement) {
+          function createCheckerboardPattern() {
+            const pattern = document.createElement('canvas');
+            pattern.width = pattern.width / Math.floor(pattern.width / 24);
+            pattern.height = pattern.height / Math.floor(pattern.height / 24);
+            const context = pattern.getContext('2d')!;
+            context.fillStyle = 'lightgray';
+            context.fillRect(0, 0, pattern.width, pattern.height);
+            context.fillStyle = 'white';
+            context.fillRect(0, 0, pattern.width / 2, pattern.height / 2);
+            context.fillRect(pattern.width / 2, pattern.height / 2, pattern.width, pattern.height);
+            return context.createPattern(pattern, 'repeat')!;
+          }
+
+          context.fillStyle = createCheckerboardPattern();
+          context.fillRect(0, 0, canvas.width, canvas.height);
+        }
+
+
+        if (!isTopFrame) {
+          for (const canvas of canvasElements) {
+            const context = canvas.getContext('2d')!;
+            drawCheckerboard(context, canvas);
+            canvas.title = `Playwright displays canvas contents on a best-effort basis. It doesn't support canvas elements inside an iframe yet. If this impacts your workflow, please open an issue so we can prioritize.`;
+          }
+          return;
+        }
+
+        const img = new Image();
+        img.onload = () => {
+          for (const canvas of canvasElements) {
+            const context = canvas.getContext('2d')!;
+
+            const boundingRect = canvas.getBoundingClientRect();
+            const xStart = boundingRect.left / window.innerWidth;
+            const yStart = boundingRect.top / window.innerHeight;
+            const xEnd = boundingRect.right / window.innerWidth;
+            const yEnd = boundingRect.bottom / window.innerHeight;
+
+            const partiallyUncaptured = xEnd > 1 || yEnd > 1;
+            const fullyUncaptured = xStart > 1 || yStart > 1;
+            if (fullyUncaptured) {
+              canvas.title = `Playwright couldn't capture canvas contents because it's located outside the viewport.`;
+              continue;
+            }
+
+            drawCheckerboard(context, canvas);
+
+            context.drawImage(img, xStart * img.width, yStart * img.height, (xEnd - xStart) * img.width, (yEnd - yStart) * img.height, 0, 0, canvas.width, canvas.height);
+            if (isUnderTest)
+              // eslint-disable-next-line no-console
+              console.log(`canvas drawn:`, JSON.stringify([xStart, yStart, xEnd, yEnd].map(v => Math.floor(v * 100))));
+
+            if (partiallyUncaptured)
+              canvas.title = `Playwright couldn't capture full canvas contents because it's located partially outside the viewport.`;
+            else
+              canvas.title = `Canvas contents are displayed on a best-effort basis based on viewport screenshots taken during test execution.`;
+          }
+        };
+        img.onerror = () => {
+          for (const canvas of canvasElements) {
+            const context = canvas.getContext('2d')!;
+            drawCheckerboard(context, canvas);
+            canvas.title = `Playwright couldn't show canvas contents because the screenshot failed to load.`;
+          }
+        };
+        img.src = location.href.replace('/snapshot', '/closest-screenshot');
+      }
     };
 
     const onDOMContentLoaded = () => visit(document);

packages/trace-viewer/src/sw/snapshotServer.ts

@@ -35,11 +35,20 @@ export class SnapshotServer {
     const snapshot = this._snapshot(pathname.substring('/snapshot'.length), searchParams);
     if (!snapshot)
       return new Response(null, { status: 404 });
+
     const renderedSnapshot = snapshot.render();
     this._snapshotIds.set(snapshotUrl, snapshot);
     return new Response(renderedSnapshot.html, { status: 200, headers: { 'Content-Type': 'text/html; charset=utf-8' } });
   }
 
+  async serveClosestScreenshot(pathname: string, searchParams: URLSearchParams): Promise<Response> {
+    const snapshot = this._snapshot(pathname.substring('/closest-screenshot'.length), searchParams);
+    const sha1 = snapshot?.closestScreenshot();
+    if (!sha1)
+      return new Response(null, { status: 404 });
+    return new Response(await this._resourceLoader(sha1));
+  }
+
   serveSnapshotInfo(pathname: string, searchParams: URLSearchParams): Response {
     const snapshot = this._snapshot(pathname.substring('/snapshotInfo'.length), searchParams);
     return this._respondWithJson(snapshot ? {

packages/trace-viewer/src/sw/snapshotStorage.ts

@@ -16,6 +16,7 @@
 
 import type { FrameSnapshot, ResourceSnapshot } from '@trace/snapshot';
 import { rewriteURLForCustomProtocol, SnapshotRenderer } from './snapshotRenderer';
+import type { PageEntry } from '../types/entries';
 
 export class SnapshotStorage {
   private _resources: ResourceSnapshot[] = [];
@@ -29,7 +30,7 @@ export class SnapshotStorage {
     this._resources.push(resource);
   }
 
-  addFrameSnapshot(snapshot: FrameSnapshot) {
+  addFrameSnapshot(snapshot: FrameSnapshot, screencastFrames: PageEntry['screencastFrames']) {
     for (const override of snapshot.resourceOverrides)
       override.url = rewriteURLForCustomProtocol(override.url);
     let frameSnapshots = this._frameSnapshots.get(snapshot.frameId);
@@ -43,7 +44,7 @@ export class SnapshotStorage {
         this._frameSnapshots.set(snapshot.pageId, frameSnapshots);
     }
     frameSnapshots.raw.push(snapshot);
-    const renderer = new SnapshotRenderer(this._resources, frameSnapshots.raw, frameSnapshots.raw.length - 1);
+    const renderer = new SnapshotRenderer(this._resources, frameSnapshots.raw, screencastFrames, frameSnapshots.raw.length - 1);
     frameSnapshots.renderers.push(renderer);
     return renderer;
   }

packages/trace-viewer/src/sw/traceModernizer.ts

@@ -159,7 +159,7 @@ export class TraceModernizer {
         contextEntry.resources.push(event.snapshot);
         break;
       case 'frame-snapshot':
-        this._snapshotStorage.addFrameSnapshot(event.snapshot);
+        this._snapshotStorage.addFrameSnapshot(event.snapshot, this._pageEntry(event.snapshot.pageId).screencastFrames);
         break;
     }
     // Make sure there is a page entry for each page, even without screencast frames,

packages/trace-viewer/src/ui/snapshotTab.tsx

@@ -369,10 +369,14 @@ export function collectSnapshots(action: ActionTraceEvent | undefined): Snapshot
   return { action: actionSnapshot, before: beforeSnapshot, after: afterSnapshot };
 }
 
+const isUnderTest = new URLSearchParams(window.location.search).has('isUnderTest');
+
 export function extendSnapshot(snapshot: Snapshot): SnapshotUrls {
   const params = new URLSearchParams();
   params.set('trace', context(snapshot.action).traceUrl);
   params.set('name', snapshot.snapshotName);
+  if (isUnderTest)
+    params.set('isUnderTest', 'true');
   if (snapshot.point) {
     params.set('pointX', String(snapshot.point.x));
     params.set('pointY', String(snapshot.point.y));

tests/assets/screenshots/canvas.html

@@ -7,4 +7,7 @@
   ctx.fillRect(25, 25, 100, 100);
   ctx.clearRect(45, 45, 60, 60);
   ctx.strokeRect(50, 50, 50, 50);
+
+  if (location.hash.includes('canvas-on-edge'))
+    canvas.style.marginTop = '90vh';
 </script>

tests/library/trace-viewer.spec.ts

@@ -21,6 +21,7 @@ import path from 'path';
 import { pathToFileURL } from 'url';
 import { expect, playwrightTest } from '../config/browserTest';
 import type { FrameLocator } from '@playwright/test';
+import { rafraf } from 'tests/page/pageTest';
 
 const test = playwrightTest.extend<TraceViewerFixtures>(traceViewerFixtures);
 
@@ -1439,6 +1440,32 @@ test.skip('should allow showing screenshots instead of snapshots', async ({ runA
   await expect(screenshot).toBeVisible();
 });
 
+test('canvas clipping', async ({ runAndTrace, page, server }) => {
+  const traceViewer = await runAndTrace(async () => {
+    await page.goto(server.PREFIX + '/screenshots/canvas.html#canvas-on-edge');
+    await rafraf(page, 5);
+  });
+
+  const msg = await traceViewer.page.waitForEvent('console', { predicate: msg => msg.text().startsWith('canvas drawn:') });
+  expect(msg.text()).toEqual('canvas drawn: [0,91,12,111]');
+
+  const snapshot = await traceViewer.snapshotFrame('page.goto');
+  await expect(snapshot.locator('canvas')).toHaveAttribute('title', `Playwright couldn't capture full canvas contents because it's located partially outside the viewport.`);
+});
+
+test('canvas clipping in iframe', async ({ runAndTrace, page, server }) => {
+  const traceViewer = await runAndTrace(async () => {
+    await page.setContent(`
+      <iframe src="${server.PREFIX}/screenshots/canvas.html#canvas-on-edge"></iframe>
+    `);
+    await rafraf(page, 5);
+  });
+
+  const snapshot = await traceViewer.snapshotFrame('page.evaluate');
+  const canvas = snapshot.locator('iframe').contentFrame().locator('canvas');
+  await expect(canvas).toHaveAttribute('title', `Playwright displays canvas contents on a best-effort basis. It doesn't support canvas elements inside an iframe yet. If this impacts your workflow, please open an issue so we can prioritize.`);
+});
+
 test.skip('should handle case where neither snapshots nor screenshots exist', async ({ runAndTrace, page, server }) => {
   const traceViewer = await runAndTrace(async () => {
     await page.goto(server.PREFIX + '/one-style.html');