kyfx
/
playwright
mirror of https://github.com/microsoft/playwright.git
4
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

cherry-pick(#35032): chore: improve prompt to use code frame and inline error

This commit is contained in:
Pavel Feldman 2025-03-04 19:28:03 -08:00
parent 230da2927d
commit 7cccd68de6
8 changed files with 172 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/todomvc/playwright.config.ts
View File

@ -12,6 +12,8 @@ export default defineConfig({
/* Maximum time one test can run for. */
timeout: 15_000,
captureGitInfo: { commit: true, diff: true },
expect: {
/**

28
packages/playwright-core/src/utils/isomorphic/stackTrace.ts
View File

@ -134,6 +134,34 @@ export function splitErrorMessage(message: string): { name: string, message: str
};
}
export function parseErrorStack(stack: string, pathSeparator: string, showInternalStackFrames: boolean = false): {
message: string;
stackLines: string[];
location?: StackFrame;
} {
const lines = stack.split('\n');
let firstStackLine = lines.findIndex(line => line.startsWith(' at '));
if (firstStackLine === -1)
firstStackLine = lines.length;
const message = lines.slice(0, firstStackLine).join('\n');
const stackLines = lines.slice(firstStackLine);
let location: StackFrame | undefined;
for (const line of stackLines) {
const frame = parseStackFrame(line, pathSeparator, showInternalStackFrames);
if (!frame || !frame.file)
continue;
if (belongsToNodeModules(frame.file, pathSeparator))
continue;
location = { file: frame.file, column: frame.column || 0, line: frame.line || 0 };
break;
}
return { message, stackLines, location };
}
function belongsToNodeModules(file: string, pathSeparator: string) {
return file.includes(`${pathSeparator}node_modules${pathSeparator}`);
}
const re = new RegExp('^' +
// Sometimes we strip out the ' at' because it's noisy
'(?:\\s*at )?' +

4
packages/playwright/src/DEPS.list
View File

@ -8,7 +8,11 @@ common/
[index.ts]
@testIsomorphic/**
./prompt.ts
./worker/testTracing.ts
[internalsForTest.ts]
**
[prompt.ts]
./transform/babelBundle.ts

71
packages/playwright/src/index.ts
View File

@ -22,15 +22,15 @@ import { setBoxedStackPrefixes, asLocator, createGuid, currentZone, debugMode, i
import { currentTestInfo } from './common/globals';
import { rootTestType } from './common/testType';
import { stripAnsiEscapes } from './util';
import { attachErrorPrompts } from './prompt';
import type { MetadataWithCommitInfo } from './isomorphic/types';
import type { Fixtures, PlaywrightTestArgs, PlaywrightTestOptions, PlaywrightWorkerArgs, PlaywrightWorkerOptions, ScreenshotMode, TestInfo, TestType, VideoMode } from '../types/test';
import type { ContextReuseMode } from './common/config';
import type { TestInfoImpl, TestStepInternal } from './worker/testInfo';
import type { ApiCallData, ClientInstrumentation, ClientInstrumentationListener } from '../../playwright-core/src/client/clientInstrumentation';
import type { Playwright as PlaywrightImpl } from '../../playwright-core/src/client/playwright';
import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions, LaunchOptions, Page, Tracing, Video } from 'playwright-core';
export { expect } from './matchers/expect';
export const _baseTest: TestType<{}, {}> = rootTestType.test;
@ -619,6 +619,7 @@ class ArtifactsRecorder {
private _screenshotRecorder: SnapshotRecorder;
private _pageSnapshot: string | undefined;
private _sourceCache: Map<string, string> = new Map();
constructor(playwright: PlaywrightImpl, artifactsDir: string, screenshot: ScreenshotOption) {
this._playwright = playwright;
@ -701,71 +702,7 @@ class ArtifactsRecorder {
})));
await this._screenshotRecorder.persistTemporary();
await this._attachErrorPrompts();
}
private async _attachErrorPrompts() {
if (process.env.PLAYWRIGHT_NO_COPY_PROMPT)
return;
if (this._testInfo.errors.length === 0)
return;
const testSources = await fs.promises.readFile(this._testInfo.file, 'utf-8');
for (const [index, error] of this._testInfo.errors.entries()) {
if (this._testInfo.attachments.find(a => a.name === `_prompt-${index}`))
continue;
const metadata = this._testInfo.config.metadata as MetadataWithCommitInfo;
const promptParts = [
`My Playwright test failed.`,
`Explain why, be concise, respect Playwright best practices.`,
'',
`Failed test: ${this._testInfo.titlePath.join(' >> ')}`,
'',
'Error:',
'',
'```',
stripAnsiEscapes(error.stack || error.message || ''),
'```',
];
if (this._pageSnapshot) {
promptParts.push(
'',
'Page snapshot:',
'```yaml',
this._pageSnapshot,
'```',
);
}
if (metadata.gitDiff) {
promptParts.push(
'',
'Local changes:',
'```diff',
metadata.gitDiff,
'```',
);
}
promptParts.push(
'',
'Test file:',
'```ts',
`// ${this._testInfo.file}`,
testSources,
'```',
);
this._testInfo._attach({
name: `_prompt-${index}`,
contentType: 'text/markdown',
body: Buffer.from(promptParts.join('\n')),
}, undefined);
}
await attachErrorPrompts(this._testInfo, this._sourceCache, this._pageSnapshot);
}
private async _startTraceChunkOnContextCreation(tracing: Tracing) {

124
packages/playwright/src/prompt.ts Normal file
View File

@ -0,0 +1,124 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the 'License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as fs from 'fs';
import * as path from 'path';
import { parseErrorStack } from 'playwright-core/lib/utils';
import { stripAnsiEscapes } from './util';
import { codeFrameColumns } from './transform/babelBundle';
import type { TestInfo } from '../types/test';
import type { MetadataWithCommitInfo } from './isomorphic/types';
import type { TestInfoImpl } from './worker/testInfo';
export async function attachErrorPrompts(testInfo: TestInfo, sourceCache: Map<string, string>, ariaSnapshot: string | undefined) {
if (process.env.PLAYWRIGHT_NO_COPY_PROMPT)
return;
for (const [index, error] of testInfo.errors.entries()) {
if (testInfo.attachments.find(a => a.name === `_prompt-${index}`))
continue;
const metadata = testInfo.config.metadata as MetadataWithCommitInfo;
const promptParts = [
`# Instructions`,
'',
`- Following Playwright test failed.`,
`- Explain why, be concise, respect Playwright best practices.`,
`- Provide a snippet of code with the fix is possible.`,
'',
`# Test info`,
'',
`- Name: ${testInfo.titlePath.slice(1).join(' >> ')}`,
`- Location: ${testInfo.file}:${testInfo.line}:${testInfo.column}`,
'',
'# Error details',
'',
'```',
stripAnsiEscapes(error.stack || error.message || ''),
'```',
];
if (ariaSnapshot) {
promptParts.push(
'',
'# Page snapshot',
'',
'```yaml',
ariaSnapshot,
'```',
);
}
const parsedError = error.stack ? parseErrorStack(error.stack, path.sep) : undefined;
const inlineMessage = stripAnsiEscapes(parsedError?.message || error.message || '').split('\n')[0];
const location = parsedError?.location || { file: testInfo.file, line: testInfo.line, column: testInfo.column };
const source = await loadSource(location.file, sourceCache);
const codeFrame = codeFrameColumns(
source,
{
start: {
line: location.line,
column: location.column
},
},
{
highlightCode: false,
linesAbove: 100,
linesBelow: 100,
message: inlineMessage || undefined,
}
);
promptParts.push(
'',
'# Test source',
'',
'```ts',
codeFrame,
'```',
);
if (metadata.gitDiff) {
promptParts.push(
'',
'# Local changes',
'',
'```diff',
metadata.gitDiff,
'```',
);
}
(testInfo as TestInfoImpl)._attach({
name: `_prompt-${index}`,
contentType: 'text/markdown',
body: Buffer.from(promptParts.join('\n')),
}, undefined);
}
}
async function loadSource(file: string, sourceCache: Map<string, string>) {
let source = sourceCache.get(file);
if (!source) {
// A mild race is Ok here.
source = await fs.promises.readFile(file, 'utf8');
sourceCache.set(file, source);
}
return source;
}

25
packages/playwright/src/reporters/base.ts
View File

@ -16,8 +16,7 @@
import path from 'path';
import { getPackageManagerExecCommand } from 'playwright-core/lib/utils';
import { parseStackFrame } from 'playwright-core/lib/utils';
import { getPackageManagerExecCommand, parseErrorStack } from 'playwright-core/lib/utils';
import { ms as milliseconds } from 'playwright-core/lib/utilsBundle';
import { colors as realColors, noColors } from 'playwright-core/lib/utils';
@ -540,23 +539,7 @@ export function prepareErrorStack(stack: string): {
stackLines: string[];
location?: Location;
} {
const lines = stack.split('\n');
let firstStackLine = lines.findIndex(line => line.startsWith(' at '));
if (firstStackLine === -1)
firstStackLine = lines.length;
const message = lines.slice(0, firstStackLine).join('\n');
const stackLines = lines.slice(firstStackLine);
let location: Location | undefined;
for (const line of stackLines) {
const frame = parseStackFrame(line, path.sep, !!process.env.PWDEBUGIMPL);
if (!frame || !frame.file)
continue;
if (belongsToNodeModules(frame.file))
continue;
location = { file: frame.file, column: frame.column || 0, line: frame.line || 0 };
break;
}
return { message, stackLines, location };
return parseErrorStack(stack, path.sep, !!process.env.PWDEBUGIMPL);
}
function characterWidth(c: string) {
@ -611,10 +594,6 @@ export function fitToWidth(line: string, width: number, prefix?: string): string
return taken.reverse().join('');
}
function belongsToNodeModules(file: string) {
return file.includes(`${path.sep}node_modules${path.sep}`);
}
function resolveFromEnv(name: string): string | undefined {
const value = process.env[name];
if (value)

4
tests/playwright-test/ui-mode-llm.spec.ts
View File

@ -18,7 +18,7 @@ import { test, expect, retries } from './ui-mode-fixtures';
test.describe.configure({ mode: 'parallel', retries });
test('openai', async ({ runUITest, server }) => {
test.fixme('openai', async ({ runUITest, server }) => {
server.setRoute('/v1/chat/completions', async (req, res) => {
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Access-Control-Allow-Headers', '*');
@ -59,7 +59,7 @@ test('openai', async ({ runUITest, server }) => {
`);
});
test('anthropic', async ({ runUITest, server }) => {
test.fixme('anthropic', async ({ runUITest, server }) => {
server.setRoute('/v1/messages', async (req, res) => {
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Access-Control-Allow-Headers', '*');

10
tests/playwright-test/ui-mode-trace.spec.ts
View File

@ -519,9 +519,11 @@ test('fails', async ({ page }) => {
const prompt = await page.evaluate(() => navigator.clipboard.readText());
expect(prompt, 'contains error').toContain('expect(received).toBe(expected)');
expect(prompt.replaceAll('\r\n', '\n'), 'contains test sources').toContain(`
test('fails', async ({ page }) => {
await page.setContent('<button>Submit</button>');
expect(1).toBe(2);
});
1 | import { test, expect } from '@playwright/test';
2 | test('fails', async ({ page }) => {
3 | await page.setContent('<button>Submit</button>');
> 4 | expect(1).toBe(2);
| ^ Error: expect(received).toBe(expected) // Object.is equality
5 | });
`.trim());
});