feat(html): "copy prompt" button (#34670)

packages/html-reporter/src/metadataView.tsx

@@ -26,9 +26,25 @@ import { linkifyText } from '@web/renderUtils';
 
 type MetadataEntries = [string, unknown][];
 
-export function filterMetadata(metadata: Metadata): MetadataEntries {
+export const MetadataContext = React.createContext<MetadataEntries>([]);
+
+export function MetadataProvider({ metadata, children }: React.PropsWithChildren<{ metadata: Metadata }>) {
+  const entries = React.useMemo(() => {
     // TODO: do not plumb actualWorkers through metadata.
+
     return Object.entries(metadata).filter(([key]) => key !== 'actualWorkers');
+  }, [metadata]);
+
+  return <MetadataContext.Provider value={entries}>{children}</MetadataContext.Provider>;
+}
+
+export function useMetadata() {
+  return React.useContext(MetadataContext);
+}
+
+export function useGitCommitInfo() {
+  const metadataEntries = useMetadata();
+  return metadataEntries.find(([key]) => key === 'git.commit.info')?.[1] as GitCommitInfo | undefined;
 }
 
 class ErrorBoundary extends React.Component<React.PropsWithChildren<{}>, { error: Error | null, errorInfo: React.ErrorInfo | null }> {
@@ -57,12 +73,13 @@ class ErrorBoundary extends React.Component<React.PropsWithChildren<{}>, { error
   }
 }
 
-export const MetadataView: React.FC<{ metadataEntries: MetadataEntries }> = ({ metadataEntries }) => {
-  return <ErrorBoundary><InnerMetadataView metadataEntries={metadataEntries}/></ErrorBoundary>;
+export const MetadataView = () => {
+  return <ErrorBoundary><InnerMetadataView/></ErrorBoundary>;
 };
 
-const InnerMetadataView: React.FC<{ metadataEntries: MetadataEntries }> = ({ metadataEntries }) => {
-  const gitCommitInfo = metadataEntries.find(([key]) => key === 'git.commit.info')?.[1] as GitCommitInfo | undefined;
+const InnerMetadataView = () => {
+  const metadataEntries = useMetadata();
+  const gitCommitInfo = useGitCommitInfo();
   const entries = metadataEntries.filter(([key]) => key !== 'git.commit.info');
   if (!gitCommitInfo && !entries.length)
     return null;

packages/html-reporter/src/reportView.tsx

@@ -26,6 +26,7 @@ import './reportView.css';
 import { TestCaseView } from './testCaseView';
 import { TestFilesHeader, TestFilesView } from './testFilesView';
 import './theme.css';
+import { MetadataProvider } from './metadataView';
 
 declare global {
   interface Window {
@@ -72,7 +73,7 @@ export const ReportView: React.FC<{
     return result;
   }, [report, filter]);
 
-  return <div className='htmlreport vbox px-4 pb-4'>
+  return <MetadataProvider metadata={report?.json().metadata ?? {}}><div className='htmlreport vbox px-4 pb-4'>
     <main>
       {report?.json() && <HeaderView stats={report.json().stats} filterText={filterText} setFilterText={setFilterText}></HeaderView>}
       <Route predicate={testFilesRoutePredicate}>
@@ -88,7 +89,7 @@ export const ReportView: React.FC<{
         {!!report && <TestCaseViewLoader report={report} tests={filteredTests.tests} testIdToFileIdMap={testIdToFileIdMap} />}
       </Route>
     </main>
-  </div>;
+  </div></MetadataProvider>;
 };
 
 const TestCaseViewLoader: React.FC<{

packages/html-reporter/src/testErrorView.css

@@ -16,18 +16,47 @@
 
 @import '@web/third_party/vscode/colors.css';
 
-.test-error-view {
+.test-error-container {
   white-space: pre;
   overflow: auto;
   flex: none;
   padding: 0;
   background-color: var(--color-canvas-subtle);
   border-radius: 6px;
-  padding: 16px;
   line-height: initial;
   margin-bottom: 6px;
 }
 
+.test-error-view {
+  padding: 16px;
+}
+
 .test-error-text {
   font-family: monospace;
 }
+
+.prompt-button {
+  flex: none;
+  height: 24px;
+  width: 80px;
+  border: 1px solid var(--color-btn-border);
+  outline: none;
+  color: var(--color-btn-text);
+  background: var(--color-btn-bg);
+  padding: 4px;
+  cursor: pointer;
+  display: inline-flex;
+  align-items: center;
+  justify-content: center;
+  border-radius: 4px;
+}
+
+.prompt-button svg {
+  color: var(--color-fg-subtle);
+}
+
+.prompt-button:not(:disabled):hover {
+  border-color: var(--color-btn-hover-border);
+  background-color: var(--color-btn-hover-bg);
+}
+

packages/html-reporter/src/testErrorView.tsx

@@ -17,15 +17,57 @@
 import { ansi2html } from '@web/ansi2html';
 import * as React from 'react';
 import './testErrorView.css';
+import * as icons from './icons';
 import type { ImageDiff } from '@web/shared/imageDiffView';
 import { ImageDiffView } from '@web/shared/imageDiffView';
+import type { TestResult } from './types';
+import { fixTestPrompt } from '@web/components/prompts';
+import { useGitCommitInfo } from './metadataView';
 
-export const TestErrorView: React.FC<{
+export const TestErrorView: React.FC<{ error: string; testId?: string; result?: TestResult }> = ({ error, testId, result }) => {
+  return (
+    <CodeSnippet code={error} testId={testId}>
+      <div style={{ float: 'right', padding: '5px' }}>
+        <PromptButton error={error} result={result} />
+      </div>
+    </CodeSnippet>
+  );
+};
+
+export const CodeSnippet = ({ code, children, testId }: React.PropsWithChildren<{ code: string; testId?: string; }>) => {
+  const html = React.useMemo(() => ansiErrorToHtml(code), [code]);
+  return (
+    <div className='test-error-container test-error-text' data-testid={testId}>
+      {children}
+      <div className='test-error-view' dangerouslySetInnerHTML={{ __html: html || '' }}></div>
+    </div>
+  );
+};
+
+const PromptButton: React.FC<{
   error: string;
-  testId?: string;
-}> = ({ error, testId }) => {
-  const html = React.useMemo(() => ansiErrorToHtml(error), [error]);
-  return <div className='test-error-view test-error-text' data-testid={testId} dangerouslySetInnerHTML={{ __html: html || '' }}></div>;
+  result?: TestResult;
+}> = ({ error, result }) => {
+  const gitCommitInfo = useGitCommitInfo();
+  const prompt = React.useMemo(() => fixTestPrompt(
+      error,
+      gitCommitInfo?.['pull.diff'] ?? gitCommitInfo?.['revision.diff'],
+      result?.attachments.find(a => a.name === 'pageSnapshot')?.body
+  ), [gitCommitInfo, result, error]);
+
+  const [copied, setCopied] = React.useState(false);
+
+  return <button
+    className='prompt-button'
+    onClick={async () => {
+      await navigator.clipboard.writeText(prompt);
+      setCopied(true);
+      setTimeout(() => {
+        setCopied(false);
+      }, 3000);
+    }}>
+    {copied ? <span className='prompt-button-copied'>Copied <icons.copy/></span> : 'Fix with AI'}
+  </button>;
 };
 
 export const TestScreenshotErrorView: React.FC<{

packages/html-reporter/src/testFilesView.tsx

@@ -22,7 +22,7 @@ import { msToString } from './utils';
 import { AutoChip } from './chip';
 import { TestErrorView } from './testErrorView';
 import * as icons from './icons';
-import { filterMetadata, MetadataView } from './metadataView';
+import { MetadataView, useMetadata } from './metadataView';
 
 export const TestFilesView: React.FC<{
   tests: TestFileSummary[],
@@ -67,9 +67,9 @@ export const TestFilesHeader: React.FC<{
   metadataVisible: boolean,
   toggleMetadataVisible: () => void,
 }> = ({ report, filteredStats, metadataVisible, toggleMetadataVisible }) => {
+  const metadataEntries = useMetadata();
   if (!report)
     return;
-  const metadataEntries = filterMetadata(report.metadata || {});
   return <>
     <div className='mx-1' style={{ display: 'flex', marginTop: 10 }}>
       {metadataEntries.length > 0 && <div className='metadata-toggle' role='button' onClick={toggleMetadataVisible} title={metadataVisible ? 'Hide metadata' : 'Show metadata'}>
@@ -81,7 +81,7 @@ export const TestFilesHeader: React.FC<{
       <div data-testid='overall-time' style={{ color: 'var(--color-fg-subtle)', marginRight: '10px' }}>{report ? new Date(report.startTime).toLocaleString() : ''}</div>
       <div data-testid='overall-duration' style={{ color: 'var(--color-fg-subtle)' }}>Total time: {msToString(report.duration ?? 0)}</div>
     </div>
-    {metadataVisible && <MetadataView metadataEntries={metadataEntries}/>}
+    {metadataVisible && <MetadataView/>}
     {!!report.errors.length && <AutoChip header='Errors' dataTestId='report-errors'>
       {report.errors.map((error, index) => <TestErrorView key={'test-report-error-message-' + index} error={error}></TestErrorView>)}
     </AutoChip>}

packages/html-reporter/src/testResultView.tsx

@@ -24,7 +24,7 @@ import { Anchor, AttachmentLink, generateTraceUrl, testResultHref } from './link
 import { statusIcon } from './statusIcon';
 import type { ImageDiff } from '@web/shared/imageDiffView';
 import { ImageDiffView } from '@web/shared/imageDiffView';
-import { TestErrorView, TestScreenshotErrorView } from './testErrorView';
+import { CodeSnippet, TestErrorView, TestScreenshotErrorView } from './testErrorView';
 import * as icons from './icons';
 import './testResultView.css';
 
@@ -90,7 +90,7 @@ export const TestResultView: React.FC<{
       {errors.map((error, index) => {
         if (error.type === 'screenshot')
           return <TestScreenshotErrorView key={'test-result-error-message-' + index} errorPrefix={error.errorPrefix} diff={error.diff!} errorSuffix={error.errorSuffix}></TestScreenshotErrorView>;
-        return <TestErrorView key={'test-result-error-message-' + index} error={error.error!}></TestErrorView>;
+        return <TestErrorView key={'test-result-error-message-' + index} error={error.error!} result={result}></TestErrorView>;
       })}
     </AutoChip>}
     {!!result.steps.length && <AutoChip header='Test Steps'>
@@ -182,7 +182,7 @@ const StepTreeItem: React.FC<{
     {step.count > 1 && <> ✕ <span className='test-result-counter'>{step.count}</span></>}
     {step.location && <span className='test-result-path'>— {step.location.file}:{step.location.line}</span>}
   </span>} loadChildren={step.steps.length || step.snippet ? () => {
-    const snippet = step.snippet ? [<TestErrorView testId='test-snippet' key='line' error={step.snippet}/>] : [];
+    const snippet = step.snippet ? [<CodeSnippet testId='test-snippet' key='line' code={step.snippet} />] : [];
     const steps = step.steps.map((s, i) => <StepTreeItem key={i} step={s} depth={depth + 1} result={result} test={test} />);
     return snippet.concat(steps);
   } : undefined} depth={depth}/>;

packages/playwright/src/reporters/html.ts

@@ -449,6 +449,17 @@ class HtmlBuilder {
         return a;
       }
 
+      if (a.name === 'pageSnapshot') {
+        try {
+          const body = fs.readFileSync(a.path!, { encoding: 'utf-8' });
+          return {
+            name: 'pageSnapshot',
+            contentType: a.contentType,
+            body,
+          };
+        } catch {}
+      }
+
       if (a.path) {
         let fileName = a.path;
         try {

packages/web/src/components/prompts.ts

@@ -0,0 +1,48 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const ansiRegex = new RegExp('([\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~])))', 'g');
+function stripAnsiEscapes(str: string): string {
+  return str.replace(ansiRegex, '');
+}
+
+export function fixTestPrompt(error: string, diff?: string, pageSnapshot?: string) {
+  const promptParts = [
+    'This test failed, suggest how to fix it. Please be correct, concise and keep Playwright best practices in mind.',
+    'Here is the error:',
+    '\n',
+    stripAnsiEscapes(error),
+    '\n',
+  ];
+
+  if (pageSnapshot) {
+    promptParts.push(
+        'This is how the page looked at the end of the test:',
+        pageSnapshot,
+        '\n'
+    );
+  }
+
+  if (diff) {
+    promptParts.push(
+        'And this is the code diff:',
+        diff,
+        '\n'
+    );
+  }
+
+  return promptParts.join('\n');
+}

tests/playwright-test/reporter-html.spec.ts

@@ -2694,6 +2694,59 @@ for (const useIntermediateMergeReport of [true, false] as const) {
       await page.getByText('my test').click();
       await expect(page.locator('.tree-item', { hasText: 'stdout' })).toHaveCount(1);
     });
+
+    test('should show AI prompt', async ({ runInlineTest, writeFiles, showReport, page }) => {
+      const files = {
+        'uncommitted.txt': `uncommitted file`,
+        'playwright.config.ts': `
+          export default {
+            populateGitInfo: true,
+            metadata: { foo: 'value1', bar: { prop: 'value2' }, baz: ['value3', 123] }
+          };
+        `,
+        'example.spec.ts': `
+          import { test, expect } from '@playwright/test';
+          test('sample', async ({}) => { expect(2).toBe(3); });
+        `,
+      };
+      const baseDir = await writeFiles(files);
+
+      const execGit = async (args: string[]) => {
+        const { code, stdout, stderr } = await spawnAsync('git', args, { stdio: 'pipe', cwd: baseDir });
+        if (!!code)
+          throw new Error(`Non-zero exit of:\n$ git ${args.join(' ')}\nConsole:\nstdout:\n${stdout}\n\nstderr:\n${stderr}\n\n`);
+        return;
+      };
+
+      await execGit(['init']);
+      await execGit(['config', '--local', 'user.email', 'shakespeare@example.local']);
+      await execGit(['config', '--local', 'user.name', 'William']);
+      await execGit(['add', 'playwright.config.ts']);
+      await execGit(['commit', '-m', 'init']);
+      await execGit(['add', '*.ts']);
+      await execGit(['commit', '-m', 'chore(html): make this test look nice']);
+
+      const result = await runInlineTest(files, { reporter: 'dot,html' }, {
+        PLAYWRIGHT_HTML_OPEN: 'never',
+        GITHUB_REPOSITORY: 'microsoft/playwright-example-for-test',
+        GITHUB_RUN_ID: 'example-run-id',
+        GITHUB_SERVER_URL: 'https://playwright.dev',
+        GITHUB_SHA: 'example-sha',
+        GITHUB_REF_NAME: '42/merge',
+        GITHUB_BASE_REF: 'HEAD~1',
+      });
+
+      expect(result.exitCode).toBe(1);
+      await showReport();
+
+      await page.context().grantPermissions(['clipboard-read', 'clipboard-write']);
+
+      await page.getByRole('link', { name: 'sample' }).click();
+      await page.getByRole('button', { name: 'Fix with AI' }).click();
+      const prompt = await page.evaluate(() => navigator.clipboard.readText());
+      expect(prompt, 'contains error').toContain('expect(received).toBe(expected)');
+      expect(prompt, 'contains diff').toContain(`+          test('sample', async ({}) => { expect(2).toBe(3); });`);
+    });
   });
 }