kyfx
/
playwright
mirror of https://github.com/microsoft/playwright.git
4
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

chore(step): remove step.fail and step.fixme, add step.skip (#34326)

This commit is contained in:
Yury Semikhatsky 2025-01-14 17:43:47 -08:00 committed by GitHub
parent 0869195ba4
commit 275f334b58
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 117 additions and 224 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

157
docs/src/test-api/class-test.md
View File

@ -1767,118 +1767,63 @@ Whether to box the step in the report. Defaults to `false`. When the step is box
Specifies a custom location for the step to be shown in test reports and trace viewer. By default, location of the [`method: Test.step`] call is shown.
## async method: Test.step.skip
* since: v1.50
- returns: <[void]>
Mark a test step as "skip" to temporarily disable its execution, useful for steps that are currently failing and planned for a near-term fix. Playwright will not run the step.
**Usage**
You can declare a skipped step, and Playwright will not run it.
```js
import { test, expect } from '@playwright/test';
test('my test', async ({ page }) => {
// ...
await test.step.skip('not yet ready', async () => {
// ...
});
});
```
### param: Test.step.skip.title
* since: v1.50
- `title` <[string]>
Step name.
### param: Test.step.skip.body
* since: v1.50
- `body` <[function]\(\):[Promise]<[any]>>
Step body.
### option: Test.step.skip.box
* since: v1.50
- `box` <boolean>
Whether to box the step in the report. Defaults to `false`. When the step is boxed, errors thrown from the step internals point to the step call site. See below for more details.
### option: Test.step.skip.location
* since: v1.50
- `location` <[Location]>
Specifies a custom location for the step to be shown in test reports and trace viewer. By default, location of the [`method: Test.step`] call is shown.
### option: Test.step.skip.timeout
* since: v1.50
- `timeout` <[float]>
Maximum time in milliseconds for the step to finish. Defaults to `0` (no timeout).
### option: Test.step.timeout
* since: v1.50
- `timeout` <[float]>
Maximum time in milliseconds for the step to finish. Defaults to `0` (no timeout).
## async method: Test.step.fail
* since: v1.50
- returns: <[void]>
Marks a test step as "should fail". Playwright runs this test step and ensures that it actually fails. This is useful for documentation purposes to acknowledge that some functionality is broken until it is fixed.
:::note
If the step exceeds the timeout, a [TimeoutError] is thrown. This indicates the step did not fail as expected.
:::
**Usage**
You can declare a test step as failing, so that Playwright ensures it actually fails.
```js
import { test, expect } from '@playwright/test';
test('my test', async ({ page }) => {
// ...
await test.step.fail('currently failing', async () => {
// ...
});
});
```
### param: Test.step.fail.title
* since: v1.50
- `title` <[string]>
Step name.
### param: Test.step.fail.body
* since: v1.50
- `body` <[function]\(\):[Promise]<[any]>>
Step body.
### option: Test.step.fail.box
* since: v1.50
- `box` <boolean>
Whether to box the step in the report. Defaults to `false`. When the step is boxed, errors thrown from the step internals point to the step call site. See below for more details.
### option: Test.step.fail.location
* since: v1.50
- `location` <[Location]>
Specifies a custom location for the step to be shown in test reports and trace viewer. By default, location of the [`method: Test.step`] call is shown.
### option: Test.step.fail.timeout
* since: v1.50
- `timeout` <[float]>
Maximum time in milliseconds for the step to finish. Defaults to `0` (no timeout).
## async method: Test.step.fixme
* since: v1.50
- returns: <[void]>
Mark a test step as "fixme", with the intention to fix it. Playwright will not run the step.
**Usage**
You can declare a test step as failing, so that Playwright ensures it actually fails.
```js
import { test, expect } from '@playwright/test';
test('my test', async ({ page }) => {
// ...
await test.step.fixme('not yet ready', async () => {
// ...
});
});
```
### param: Test.step.fixme.title
* since: v1.50
- `title` <[string]>
Step name.
### param: Test.step.fixme.body
* since: v1.50
- `body` <[function]\(\):[Promise]<[any]>>
Step body.
### option: Test.step.fixme.box
* since: v1.50
- `box` <boolean>
Whether to box the step in the report. Defaults to `false`. When the step is boxed, errors thrown from the step internals point to the step call site. See below for more details.
### option: Test.step.fixme.location
* since: v1.50
- `location` <[Location]>
Specifies a custom location for the step to be shown in test reports and trace viewer. By default, location of the [`method: Test.step`] call is shown.
### option: Test.step.fixme.timeout
* since: v1.50
- `timeout` <[float]>
Maximum time in milliseconds for the step to finish. Defaults to `0` (no timeout).
## method: Test.use
* since: v1.10

4
packages/html-reporter/src/testResultView.tsx
View File

@ -176,11 +176,11 @@ const StepTreeItem: React.FC<{
}> = ({ test, step, result, depth }) => {
return <TreeItem title={<span aria-label={step.title}>
<span style={{ float: 'right' }}>{msToString(step.duration)}</span>
{statusIcon(step.error || step.duration === -1 ? 'failed' : 'passed')}
{statusIcon(step.error || step.duration === -1 ? 'failed' : (step.skipped ? 'skipped' : 'passed'))}
<span>{step.title}</span>
{step.count > 1 && <> ✕ <span className='test-result-counter'>{step.count}</span></>}
{step.location && <span className='test-result-path'>— {step.location.file}:{step.location.line}</span>}
</span>} loadChildren={step.steps.length + (step.snippet ? 1 : 0) ? () => {
</span>} loadChildren={step.steps.length || step.snippet ? () => {
const snippet = step.snippet ? [<TestErrorView testId='test-snippet' key='line' error={step.snippet}/>] : [];
const steps = step.steps.map((s, i) => <StepTreeItem key={i} step={s} depth={depth + 1} result={result} test={test} />);
const attachments = step.attachments.map(attachmentIndex => (

1
packages/html-reporter/src/types.d.ts vendored
View File

@ -110,4 +110,5 @@ export type TestStep = {
steps: TestStep[];
attachments: number[];
count: number;
skipped?: boolean;
};

42
packages/playwright/src/common/testType.ts
View File

@ -57,8 +57,7 @@ export class TestTypeImpl {
test.slow = wrapFunctionWithLocation(this._modifier.bind(this, 'slow'));
test.setTimeout = wrapFunctionWithLocation(this._setTimeout.bind(this));
test.step = this._step.bind(this, 'pass');
test.step.fail = this._step.bind(this, 'fail');
test.step.fixme = this._step.bind(this, 'fixme');
test.step.skip = this._step.bind(this, 'skip');
test.use = wrapFunctionWithLocation(this._use.bind(this));
test.extend = wrapFunctionWithLocation(this._extend.bind(this));
test.info = () => {
@ -259,40 +258,27 @@ export class TestTypeImpl {
suite._use.push({ fixtures, location });
}
async _step<T>(expectation: 'pass'|'fail'|'fixme', title: string, body: () => T | Promise<T>, options: {box?: boolean, location?: Location, timeout?: number } = {}): Promise<T> {
async _step<T>(expectation: 'pass'|'skip', title: string, body: () => T | Promise<T>, options: {box?: boolean, location?: Location, timeout?: number } = {}): Promise<T> {
const testInfo = currentTestInfo();
if (!testInfo)
throw new Error(`test.step() can only be called from a test`);
if (expectation === 'fixme')
if (expectation === 'skip') {
const step = testInfo._addStep({ category: 'test.step.skip', title, location: options.location, box: options.box });
step.complete({});
return undefined as T;
}
const step = testInfo._addStep({ category: 'test.step', title, location: options.location, box: options.box });
return await zones.run('stepZone', step, async () => {
let result;
let error;
try {
result = await raceAgainstDeadline(async () => body(), options.timeout ? monotonicTime() + options.timeout : 0);
} catch (e) {
error = e;
}
if (result?.timedOut) {
const error = new errors.TimeoutError(`Step timeout ${options.timeout}ms exceeded.`);
step.complete({ error });
throw error;
}
const expectedToFail = expectation === 'fail';
if (error) {
step.complete({ error });
if (expectedToFail)
return undefined as T;
throw error;
}
if (expectedToFail) {
error = new Error(`Step is expected to fail, but passed`);
step.complete({ error });
throw error;
}
const result = await raceAgainstDeadline(async () => body(), options.timeout ? monotonicTime() + options.timeout : 0);
if (result.timedOut)
throw new errors.TimeoutError(`Step timeout ${options.timeout}ms exceeded.`);
step.complete({});
return result!.result;
return result.result;
} catch (error) {
step.complete({ error });
throw error;
}
});
}

4
packages/playwright/src/reporters/html.ts
View File

@ -517,6 +517,7 @@ class HtmlBuilder {
private _createTestStep(dedupedStep: DedupedStep, result: api.TestResult): TestStep {
const { step, duration, count } = dedupedStep;
const skipped = dedupedStep.step.category === 'test.step.skip';
const testStep: TestStep = {
title: step.title,
startTime: step.startTime.toISOString(),
@ -530,7 +531,8 @@ class HtmlBuilder {
}),
location: this._relativeLocation(step.location),
error: step.error?.message,
count
count,
skipped
};
if (step.location)
this._stepsInFile.set(step.location.file, testStep);

2
packages/playwright/src/worker/testInfo.ts
View File

@ -35,7 +35,7 @@ export interface TestStepInternal {
attachmentIndices: number[];
stepId: string;
title: string;
category: 'hook' | 'fixture' | 'test.step' | 'expect' | 'attach' | string;
category: 'hook' | 'fixture' | 'test.step' | 'test.step.skip' | 'expect' | 'attach' | string;
location?: Location;
boxedStack?: StackFrame[];
steps: TestStepInternal[];

36
packages/playwright/types/test.d.ts vendored
View File

@ -5713,18 +5713,19 @@ export interface TestType<TestArgs extends {}, WorkerArgs extends {}> {
*/
<T>(title: string, body: () => T | Promise<T>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<T>;
/**
* Mark a test step as "fixme", with the intention to fix it. Playwright will not run the step.
* Mark a test step as "skip" to temporarily disable its execution, useful for steps that are currently failing and
* planned for a near-term fix. Playwright will not run the step.
*
* **Usage**
*
* You can declare a test step as failing, so that Playwright ensures it actually fails.
* You can declare a skipped step, and Playwright will not run it.
*
* ```js
* import { test, expect } from '@playwright/test';
*
* test('my test', async ({ page }) => {
* // ...
* await test.step.fixme('not yet ready', async () => {
* await test.step.skip('not yet ready', async () => {
* // ...
* });
* });
@ -5734,34 +5735,7 @@ export interface TestType<TestArgs extends {}, WorkerArgs extends {}> {
* @param body Step body.
* @param options
*/
fixme(title: string, body: () => any | Promise<any>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<void>;
/**
* Marks a test step as "should fail". Playwright runs this test step and ensures that it actually fails. This is
* useful for documentation purposes to acknowledge that some functionality is broken until it is fixed.
*
* **NOTE** If the step exceeds the timeout, a [TimeoutError](https://playwright.dev/docs/api/class-timeouterror) is
* thrown. This indicates the step did not fail as expected.
*
* **Usage**
*
* You can declare a test step as failing, so that Playwright ensures it actually fails.
*
* ```js
* import { test, expect } from '@playwright/test';
*
* test('my test', async ({ page }) => {
* // ...
* await test.step.fail('currently failing', async () => {
* // ...
* });
* });
* ```
*
* @param title Step name.
* @param body Step body.
* @param options
*/
fail(title: string, body: () => any | Promise<any>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<void>;
skip(title: string, body: () => any | Promise<any>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<void>;
}
/**
* `expect` function can be used to create test assertions. Read more about [test assertions](https://playwright.dev/docs/test-assertions).

27
tests/playwright-test/reporter-html.spec.ts
View File

@ -757,6 +757,33 @@ for (const useIntermediateMergeReport of [true, false] as const) {
]);
});
test('should show skipped step snippets', async ({ runInlineTest, page, showReport }) => {
const result = await runInlineTest({
'playwright.config.js': `
export default { testDir: './tests' };
`,
'tests/a.test.ts': `
import { test, expect } from '@playwright/test';
test('example', async ({}) => {
await test.step.skip('skipped step title', async () => {
expect(1).toBe(1);
await test.step('inner step', async () => {
expect(1).toBe(1);
});
});
});
`,
}, { reporter: 'dot,html' }, { PLAYWRIGHT_HTML_OPEN: 'never' });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
await showReport();
await page.click('text=example');
await page.click('text=skipped step title');
await expect(page.getByTestId('test-snippet')).toContainText(`await test.step.skip('skipped step title', async () => {`);
});
test('should render annotations', async ({ runInlineTest, page, showReport }) => {
const result = await runInlineTest({
'playwright.config.js': `

54
tests/playwright-test/test-step.spec.ts
View File

@ -1499,29 +1499,26 @@ fixture | fixture: context
`);
});
test('test.step.fail and test.step.fixme should work', async ({ runInlineTest }) => {
test('test.step.skip should work', async ({ runInlineTest }) => {
const result = await runInlineTest({
'reporter.ts': stepIndentReporter,
'playwright.config.ts': `module.exports = { reporter: './reporter' };`,
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('test', async ({ }) => {
await test.step('outer step 1', async () => {
await test.step.fail('inner step 1.1', async () => {
await test.step.skip('outer step 1', async () => {
await test.step('inner step 1.1', async () => {
throw new Error('inner step 1.1 failed');
});
await test.step.fixme('inner step 1.2', async () => {});
await test.step.skip('inner step 1.2', async () => {});
await test.step('inner step 1.3', async () => {});
});
await test.step('outer step 2', async () => {
await test.step.fixme('inner step 2.1', async () => {});
await test.step.skip('inner step 2.1', async () => {});
await test.step('inner step 2.2', async () => {
expect(1).toBe(1);
});
});
await test.step.fail('outer step 3', async () => {
throw new Error('outer step 3 failed');
});
});
`
}, { reporter: '' });
@ -1531,48 +1528,16 @@ test('test.step.fail and test.step.fixme should work', async ({ runInlineTest })
expect(result.report.stats.unexpected).toBe(0);
expect(stripAnsi(result.output)).toBe(`
hook |Before Hooks
test.step |outer step 1 @ a.test.ts:4
test.step | inner step 1.1 @ a.test.ts:5
test.step | ↪ error: Error: inner step 1.1 failed
test.step | inner step 1.3 @ a.test.ts:9
test.step.skip|outer step 1 @ a.test.ts:4
test.step |outer step 2 @ a.test.ts:11
test.step.skip| inner step 2.1 @ a.test.ts:12
test.step | inner step 2.2 @ a.test.ts:13
expect | expect.toBe @ a.test.ts:14
test.step |outer step 3 @ a.test.ts:17
test.step |↪ error: Error: outer step 3 failed
hook |After Hooks
`);
});
test('timeout inside test.step.fail is an error', async ({ runInlineTest }) => {
const result = await runInlineTest({
'reporter.ts': stepIndentReporter,
'playwright.config.ts': `module.exports = { reporter: './reporter' };`,
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('test 2', async ({ }) => {
await test.step('outer step 2', async () => {
await test.step.fail('inner step 2', async () => {
await new Promise(() => {});
});
});
});
`
}, { reporter: '', timeout: 2500 });
expect(result.exitCode).toBe(1);
expect(result.report.stats.unexpected).toBe(1);
expect(stripAnsi(result.output)).toBe(`
hook |Before Hooks
test.step |outer step 2 @ a.test.ts:4
test.step | inner step 2 @ a.test.ts:5
hook |After Hooks
hook |Worker Cleanup
|Test timeout of 2500ms exceeded.
`);
});
test('skip test.step.fixme body', async ({ runInlineTest }) => {
test('skip test.step.skip body', async ({ runInlineTest }) => {
const result = await runInlineTest({
'reporter.ts': stepIndentReporter,
'playwright.config.ts': `module.exports = { reporter: './reporter' };`,
@ -1581,7 +1546,7 @@ test('skip test.step.fixme body', async ({ runInlineTest }) => {
test('test', async ({ }) => {
let didRun = false;
await test.step('outer step 2', async () => {
await test.step.fixme('inner step 2', async () => {
await test.step.skip('inner step 2', async () => {
didRun = true;
});
});
@ -1595,6 +1560,7 @@ test('skip test.step.fixme body', async ({ runInlineTest }) => {
expect(stripAnsi(result.output)).toBe(`
hook |Before Hooks
test.step |outer step 2 @ a.test.ts:5
test.step.skip| inner step 2 @ a.test.ts:6
expect |expect.toBe @ a.test.ts:10
hook |After Hooks
`);

15
tests/playwright-test/types-2.spec.ts
View File

@ -205,21 +205,14 @@ test('step should inherit return type from its callback ', async ({ runTSC }) =>
expect(result.exitCode).toBe(0);
});
test('step.fail and step.fixme return void ', async ({ runTSC }) => {
test('step.skip returns void ', async ({ runTSC }) => {
const result = await runTSC({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('test step.fail', async ({ }) => {
test('test step.skip', async ({ }) => {
// @ts-expect-error
const bad1: string = await test.step.fail('my step', () => { });
const good: void = await test.step.fail('my step', async () => {
return 2024;
});
});
test('test step.fixme', async ({ }) => {
// @ts-expect-error
const bad1: string = await test.step.fixme('my step', () => { });
const good: void = await test.step.fixme('my step', async () => {
const bad1: string = await test.step.skip('my step', () => { return ''; });
const good: void = await test.step.skip('my step', async () => {
return 2024;
});
});

3
utils/generate_types/overrides-test.d.ts vendored
View File

@ -164,8 +164,7 @@ export interface TestType<TestArgs extends {}, WorkerArgs extends {}> {
use(fixtures: Fixtures<{}, {}, TestArgs, WorkerArgs>): void;
step: {
<T>(title: string, body: () => T | Promise<T>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<T>;
fixme(title: string, body: () => any | Promise<any>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<void>;
fail(title: string, body: () => any | Promise<any>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<void>;
skip(title: string, body: () => any | Promise<any>, options?: { box?: boolean, location?: Location, timeout?: number }): Promise<void>;
}
expect: Expect<{}>;
extend<T extends {}, W extends {} = {}>(fixtures: Fixtures<T, W, TestArgs, WorkerArgs>): TestType<TestArgs & T, WorkerArgs & W>;