feat: split up static and dynamic annotations (#35292)

Signed-off-by: Simon Knott <info@simonknott.de>
Co-authored-by: Dmitry Gozman <dgozman@gmail.com>

docs/src/test-reporter-api/class-testcase.md

@@ -12,8 +12,7 @@
 
 The list of annotations applicable to the current test. Includes:
 * annotations defined on the test or suite via [`method: Test.(call)`] and [`method: Test.describe`];
-* annotations implicitly added by methods [`method: Test.skip`], [`method: Test.fixme`] and [`method: Test.fail`];
-* annotations appended to [`property: TestInfo.annotations`] during the test execution.
+* annotations implicitly added by methods [`method: Test.skip`], [`method: Test.fixme`] and [`method: Test.fail`] prior to test execution.
 
 Annotations are available during test execution through [`property: TestInfo.annotations`].
 

docs/src/test-reporter-api/class-testresult.md

@@ -14,6 +14,20 @@ A result of a single [TestCase] run.
 
 The list of files or buffers attached during the test execution through [`property: TestInfo.attachments`].
 
+## property: TestResult.annotations
+* since: v1.52
+- type: <[Array]<[Object]>>
+  - `type` <[string]> Annotation type, for example `'skip'` or `'fail'`.
+  - `description` ?<[string]> Optional description.
+
+The list of annotations appended during test execution. Includes:
+* annotations implicitly added by methods [`method: Test.skip`], [`method: Test.fixme`] and [`method: Test.fail`] during test execution;
+* annotations appended to [`property: TestInfo.annotations`].
+
+Annotations are available during test execution through [`property: TestInfo.annotations`].
+
+Learn more about [test annotations](../test-annotations.md).
+
 ## property: TestResult.duration
 * since: v1.10
 - type: <[float]>

packages/html-reporter/src/tabbedPane.tsx

@@ -30,14 +30,18 @@ export const TabbedPane: React.FunctionComponent<{
   selectedTab: string,
   setSelectedTab: (tab: string) => void
 }> = ({ tabs, selectedTab, setSelectedTab }) => {
+  const idPrefix = React.useId();
   return <div className='tabbed-pane'>
     <div className='vbox'>
       <div className='hbox' style={{ flex: 'none' }}>
-        <div className='tabbed-pane-tab-strip'>{
+        <div className='tabbed-pane-tab-strip' role='tablist'>{
           tabs.map(tab => (
             <div className={clsx('tabbed-pane-tab-element', selectedTab === tab.id && 'selected')}
               onClick={() => setSelectedTab(tab.id)}
-              key={tab.id}>
+              id={`${idPrefix}-${tab.id}`}
+              key={tab.id}
+              role='tab'
+              aria-selected={selectedTab === tab.id}>
               <div className='tabbed-pane-tab-label'>{tab.title}</div>
             </div>
           ))
@@ -46,7 +50,7 @@ export const TabbedPane: React.FunctionComponent<{
       {
         tabs.map(tab => {
           if (selectedTab === tab.id)
-            return <div key={tab.id} className='tab-content'>{tab.render()}</div>;
+            return <div key={tab.id} className='tab-content' role='tabpanel' aria-labelledby={`${idPrefix}-${tab.id}`}>{tab.render()}</div>;
         })
       }
     </div>

packages/html-reporter/src/testCaseView.spec.tsx

@@ -42,6 +42,7 @@ const result: TestResult = {
     }],
     attachments: [],
   }],
+  annotations: [],
   attachments: [],
   status: 'passed',
 };
@@ -151,6 +152,7 @@ const resultWithAttachment: TestResult = {
     name: 'attachment with inline link https://github.com/microsoft/playwright/issues/31284',
     contentType: 'text/plain'
   }],
+  annotations: [],
   status: 'passed',
 };
 
@@ -238,13 +240,15 @@ test('total duration is selected run duration', async ({ mount, page }) => {
   const component = await mount(<TestCaseView projectNames={['chromium', 'webkit']} test={testCaseWithTwoAttempts} prev={undefined} next={undefined} run={0}></TestCaseView>);
   await expect(component).toMatchAriaSnapshot(`
     - text: "My test test.spec.ts:42 200ms"
-    - text: "Run 50ms Retry #1 150ms"
+    - tablist:
+      - tab "Run 50ms"
+      - 'tab "Retry #1 150ms"'
   `);
-  await page.locator('.tabbed-pane-tab-label', { hasText: 'Run50ms' }).click();
+  await page.getByRole('tab', { name: 'Run' }).click();
   await expect(component).toMatchAriaSnapshot(`
     - text: "My test test.spec.ts:42 200ms"
   `);
-  await page.locator('.tabbed-pane-tab-label', { hasText: 'Retry #1150ms' }).click();
+  await page.getByRole('tab', { name: 'Retry' }).click();
   await expect(component).toMatchAriaSnapshot(`
     - text: "My test test.spec.ts:42 200ms"
   `);

packages/html-reporter/src/testCaseView.tsx

@@ -14,7 +14,7 @@
   limitations under the License.
 */
 
-import type { TestCase, TestCaseAnnotation, TestCaseSummary } from './types';
+import type { TestCase, TestAnnotation, TestCaseSummary } from './types';
 import * as React from 'react';
 import { TabbedPane } from './tabbedPane';
 import { AutoChip } from './chip';
@@ -46,8 +46,13 @@ export const TestCaseView: React.FC<{
   }, [test]);
 
   const visibleAnnotations = React.useMemo(() => {
-    return test?.annotations?.filter(annotation => !annotation.type.startsWith('_')) || [];
-  }, [test?.annotations]);
+    if (!test)
+      return [];
+    const annotations = [...test.annotations];
+    if (test.results[selectedResultIndex])
+      annotations.push(...test.results[selectedResultIndex].annotations);
+    return annotations.filter(annotation => !annotation.type.startsWith('_'));
+  }, [test, selectedResultIndex]);
 
   return <div className='test-case-column vbox'>
     {test && <div className='hbox'>
@@ -71,7 +76,7 @@ export const TestCaseView: React.FC<{
       {test && !!test.projectName && <ProjectLink projectNames={projectNames} projectName={test.projectName}></ProjectLink>}
       {labels && <LabelsLinkView labels={labels} />}
     </div>}
-    {!!visibleAnnotations.length && <AutoChip header='Annotations'>
+    {!!visibleAnnotations.length && <AutoChip header='Annotations' dataTestId='test-case-annotations'>
       {visibleAnnotations.map((annotation, index) => <TestCaseAnnotationView key={index} annotation={annotation} />)}
     </AutoChip>}
     {test && <TabbedPane tabs={
@@ -86,7 +91,7 @@ export const TestCaseView: React.FC<{
   </div>;
 };
 
-function TestCaseAnnotationView({ annotation: { type, description } }: { annotation: TestCaseAnnotation }) {
+function TestCaseAnnotationView({ annotation: { type, description } }: { annotation: TestAnnotation }) {
   return (
     <div className='test-case-annotation'>
       <span style={{ fontWeight: 'bold' }}>{type}</span>

packages/html-reporter/src/types.d.ts

@@ -59,7 +59,7 @@ export type TestFileSummary = {
   stats: Stats;
 };
 
-export type TestCaseAnnotation = { type: string, description?: string };
+export type TestAnnotation = { type: string, description?: string };
 
 export type TestCaseSummary = {
   testId: string,
@@ -67,7 +67,7 @@ export type TestCaseSummary = {
   path: string[];
   projectName: string;
   location: Location;
-  annotations: TestCaseAnnotation[];
+  annotations: TestAnnotation[];
   tags: string[];
   outcome: 'skipped' | 'expected' | 'unexpected' | 'flaky';
   duration: number;
@@ -98,6 +98,7 @@ export type TestResult = {
   errors: string[];
   attachments: TestAttachment[];
   status: 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted';
+  annotations: TestAnnotation[];
 };
 
 export type TestStep = {

packages/playwright/src/common/suiteUtils.ts

@@ -64,10 +64,9 @@ export function bindFileSuiteToProject(project: FullProjectInternal, suite: Suit
     // Inherit properties from parent suites.
     let inheritedRetries: number | undefined;
     let inheritedTimeout: number | undefined;
-    test.annotations = [];
     for (let parentSuite: Suite | undefined = suite; parentSuite; parentSuite = parentSuite.parent) {
       if (parentSuite._staticAnnotations.length)
-        test.annotations = [...parentSuite._staticAnnotations, ...test.annotations];
+        test.annotations.unshift(...parentSuite._staticAnnotations);
       if (inheritedRetries === undefined && parentSuite._retries !== undefined)
         inheritedRetries = parentSuite._retries;
       if (inheritedTimeout === undefined && parentSuite._timeout !== undefined)
@@ -75,7 +74,6 @@ export function bindFileSuiteToProject(project: FullProjectInternal, suite: Suit
     }
     test.retries = inheritedRetries ?? project.project.retries;
     test.timeout = inheritedTimeout ?? project.project.timeout;
-    test.annotations.push(...test._staticAnnotations);
 
     // Skip annotations imply skipped expectedStatus.
     if (test.annotations.some(a => a.type === 'skip' || a.type === 'fixme'))

packages/playwright/src/common/test.ts

@@ -262,8 +262,6 @@ export class TestCase extends Base implements reporterTypes.TestCase {
   _poolDigest = '';
   _workerHash = '';
   _projectId = '';
-  // Annotations known statically before running the test, e.g. `test.skip()` or `test(title, { annotation }, body)`.
-  _staticAnnotations: Annotation[] = [];
   // Explicitly declared tags that are not a part of the title.
   _tags: string[] = [];
 
@@ -306,7 +304,6 @@ export class TestCase extends Base implements reporterTypes.TestCase {
       requireFile: this._requireFile,
       poolDigest: this._poolDigest,
       workerHash: this._workerHash,
-      staticAnnotations: this._staticAnnotations.slice(),
       annotations: this.annotations.slice(),
       tags: this._tags.slice(),
       projectId: this._projectId,
@@ -323,7 +320,6 @@ export class TestCase extends Base implements reporterTypes.TestCase {
     test._requireFile = data.requireFile;
     test._poolDigest = data.poolDigest;
     test._workerHash = data.workerHash;
-    test._staticAnnotations = data.staticAnnotations;
     test.annotations = data.annotations;
     test._tags = data.tags;
     test._projectId = data.projectId;
@@ -351,6 +347,7 @@ export class TestCase extends Base implements reporterTypes.TestCase {
       status: 'skipped',
       steps: [],
       errors: [],
+      annotations: [],
     };
     this.results.push(result);
     return result;

packages/playwright/src/common/testType.ts

@@ -107,16 +107,16 @@ export class TestTypeImpl {
     const validatedDetails = validateTestDetails(details);
     const test = new TestCase(title, body, this, location);
     test._requireFile = suite._requireFile;
-    test._staticAnnotations.push(...validatedDetails.annotations);
+    test.annotations.push(...validatedDetails.annotations);
     test._tags.push(...validatedDetails.tags);
     suite._addTest(test);
 
     if (type === 'only' || type === 'fail.only')
       test._only = true;
     if (type === 'skip' || type === 'fixme' || type === 'fail')
-      test._staticAnnotations.push({ type });
+      test.annotations.push({ type });
     else if (type === 'fail.only')
-      test._staticAnnotations.push({ type: 'fail' });
+      test.annotations.push({ type: 'fail' });
   }
 
   private _describe(type: 'default' | 'only' | 'serial' | 'serial.only' | 'parallel' | 'parallel.only' | 'skip' | 'fixme', location: Location, titleOrFn: string | Function, fnOrDetails?: TestDetails | Function, fn?: Function) {

packages/playwright/src/isomorphic/teleReceiver.ts

@@ -68,14 +68,14 @@ export type JsonTestCase = {
   retries: number;
   tags?: string[];
   repeatEachIndex: number;
-  annotations?: { type: string, description?: string }[];
+  annotations?: Annotation[];
 };
 
 export type JsonTestEnd = {
   testId: string;
   expectedStatus: reporterTypes.TestStatus;
   timeout: number;
-  annotations: { type: string, description?: string }[];
+  annotations: Annotation[];
 };
 
 export type JsonTestResultStart = {
@@ -94,6 +94,7 @@ export type JsonTestResultEnd = {
   status: reporterTypes.TestStatus;
   errors: reporterTypes.TestError[];
   attachments: JsonAttachment[];
+  annotations?: Annotation[];
 };
 
 export type JsonTestStepStart = {
@@ -241,6 +242,8 @@ export class TeleReporterReceiver {
     result.errors = payload.errors;
     result.error = result.errors?.[0];
     result.attachments = this._parseAttachments(payload.attachments);
+    if (payload.annotations)
+      result.annotations = payload.annotations;
     this._reporter.onTestEnd?.(test, result);
     // Free up the memory as won't see these step ids.
     result._stepMap = new Map();
@@ -562,6 +565,7 @@ export class TeleTestResult implements reporterTypes.TestResult {
   stdout: reporterTypes.TestResult['stdout'] = [];
   stderr: reporterTypes.TestResult['stderr'] = [];
   attachments: reporterTypes.TestResult['attachments'] = [];
+  annotations: reporterTypes.TestResult['annotations'] = [];
   status: reporterTypes.TestStatus = 'skipped';
   steps: TeleTestStep[] = [];
   errors: reporterTypes.TestResult['errors'] = [];

packages/playwright/src/reporters/base.ts

@@ -291,10 +291,13 @@ export class TerminalReporter implements ReporterV2 {
   }
 
   private _printWarnings() {
-    const warningTests = this.suite.allTests().filter(test => test.annotations.some(a => a.type === 'warning'));
+    const warningTests = this.suite.allTests().filter(test => {
+      const annotations = [...test.annotations, ...test.results.flatMap(r => r.annotations)];
+      return annotations.some(a => a.type === 'warning');
+    });
     const encounteredWarnings = new Map<string, Array<TestCase>>();
     for (const test of warningTests) {
-      for (const annotation of test.annotations) {
+      for (const annotation of [...test.annotations, ...test.results.flatMap(r => r.annotations)]) {
         if (annotation.type !== 'warning' || annotation.description === undefined)
           continue;
         let tests = encounteredWarnings.get(annotation.description);

packages/playwright/src/reporters/html.ts

@@ -31,7 +31,7 @@ import { resolveReporterOutputPath, stripAnsiEscapes } from '../util';
 import type { ReporterV2 } from './reporterV2';
 import type { Metadata } from '../../types/test';
 import type * as api from '../../types/testReporter';
-import type { HTMLReport, Stats, TestAttachment, TestCase, TestCaseSummary, TestFile, TestFileSummary, TestResult, TestStep } from '@html-reporter/types';
+import type { HTMLReport, Stats, TestAttachment, TestCase, TestCaseSummary, TestFile, TestFileSummary, TestResult, TestStep, TestAnnotation } from '@html-reporter/types';
 import type { ZipFile } from 'playwright-core/lib/zipBundle';
 import type { TransformCallback } from 'stream';
 
@@ -404,8 +404,7 @@ class HtmlBuilder {
         projectName,
         location,
         duration,
-        // Annotations can be pushed directly, with a wrong type.
-        annotations: test.annotations.map(a => ({ type: a.type, description: a.description ? String(a.description) : a.description })),
+        annotations: this._serializeAnnotations(test.annotations),
         tags: test.tags,
         outcome: test.outcome(),
         path,
@@ -418,8 +417,7 @@ class HtmlBuilder {
         projectName,
         location,
         duration,
-        // Annotations can be pushed directly, with a wrong type.
-        annotations: test.annotations.map(a => ({ type: a.type, description: a.description ? String(a.description) : a.description })),
+        annotations: this._serializeAnnotations([...test.annotations, ...results.flatMap(r => r.annotations)]),
         tags: test.tags,
         outcome: test.outcome(),
         path,
@@ -503,6 +501,11 @@ class HtmlBuilder {
     }).filter(Boolean) as TestAttachment[];
   }
 
+  private _serializeAnnotations(annotations: api.TestCase['annotations']): TestAnnotation[] {
+    // Annotations can be pushed directly, with a wrong type.
+    return annotations.map(a => ({ type: a.type, description: a.description ? String(a.description) : a.description }));
+  }
+
   private _createTestResult(test: api.TestCase, result: api.TestResult): TestResult {
     return {
       duration: result.duration,
@@ -511,6 +514,7 @@ class HtmlBuilder {
       steps: dedupeSteps(result.steps).map(s => this._createTestStep(s, result)),
       errors: formatResultFailure(internalScreen, test, result, '').map(error => error.message),
       status: result.status,
+      annotations: this._serializeAnnotations(result.annotations),
       attachments: this._serializeAttachments([
         ...result.attachments,
         ...result.stdout.map(m => stdioAttachment(m, 'stdout')),

packages/playwright/src/reporters/json.ts

@@ -213,6 +213,7 @@ class JSONReporter implements ReporterV2 {
       retry: result.retry,
       steps: steps.length ? steps.map(s => this._serializeTestStep(s)) : undefined,
       startTime: result.startTime.toISOString(),
+      annotations: result.annotations,
       attachments: result.attachments.map(a => ({
         name: a.name,
         contentType: a.contentType,

packages/playwright/src/reporters/junit.ts

@@ -166,7 +166,8 @@ class JUnitReporter implements ReporterV2 {
       children: [] as XMLEntry[]
     };
 
-    for (const annotation of test.annotations) {
+    const annotations = [...test.annotations, ...test.results.flatMap(r => r.annotations)];
+    for (const annotation of annotations) {
       const property: XMLEntry = {
         name: 'property',
         attributes: {

packages/playwright/src/reporters/teleEmitter.ts

@@ -237,6 +237,7 @@ export class TeleReporterEmitter implements ReporterV2 {
       status: result.status,
       errors: result.errors,
       attachments: this._serializeAttachments(result.attachments),
+      annotations: result.annotations?.length ? result.annotations : undefined,
     };
   }
 

packages/playwright/src/runner/dispatcher.ts

@@ -325,8 +325,8 @@ class JobDispatcher {
     result.errors = params.errors;
     result.error = result.errors[0];
     result.status = params.status;
+    result.annotations = params.annotations;
     test.expectedStatus = params.expectedStatus;
-    test.annotations = params.annotations;
     test.timeout = params.timeout;
     const isFailure = result.status !== 'skipped' && result.status !== test.expectedStatus;
     if (isFailure)

packages/playwright/src/worker/workerMain.ts

@@ -292,6 +292,8 @@ export class WorkerMain extends ProcessRunner {
     for (const annotation of test.annotations)
       processAnnotation(annotation);
 
+    const staticAnnotations = new Set(testInfo.annotations);
+
     // Process existing annotations dynamically set for parent suites.
     for (const suite of suites) {
       const extraAnnotations = this._activeSuites.get(suite) || [];
@@ -310,7 +312,7 @@ export class WorkerMain extends ProcessRunner {
     if (isSkipped && nextTest && !hasAfterAllToRunBeforeNextTest) {
       // Fast path - this test is skipped, and there are more tests that will handle cleanup.
       testInfo.status = 'skipped';
-      this.dispatchEvent('testEnd', buildTestEndPayload(testInfo));
+      this.dispatchEvent('testEnd', buildTestEndPayload(testInfo, staticAnnotations));
       return;
     }
 
@@ -492,7 +494,7 @@ export class WorkerMain extends ProcessRunner {
 
     this._currentTest = null;
     setCurrentTestInfo(null);
-    this.dispatchEvent('testEnd', buildTestEndPayload(testInfo));
+    this.dispatchEvent('testEnd', buildTestEndPayload(testInfo, staticAnnotations));
 
     const preserveOutput = this._config.config.preserveOutput === 'always' ||
       (this._config.config.preserveOutput === 'failures-only' && testInfo._isFailure());
@@ -612,7 +614,7 @@ function buildTestBeginPayload(testInfo: TestInfoImpl): TestBeginPayload {
   };
 }
 
-function buildTestEndPayload(testInfo: TestInfoImpl): TestEndPayload {
+function buildTestEndPayload(testInfo: TestInfoImpl, staticAnnotations: Set<Annotation>): TestEndPayload {
   return {
     testId: testInfo.testId,
     duration: testInfo.duration,
@@ -620,7 +622,7 @@ function buildTestEndPayload(testInfo: TestInfoImpl): TestEndPayload {
     errors: testInfo.errors,
     hasNonRetriableError: testInfo._hasNonRetriableError,
     expectedStatus: testInfo.expectedStatus,
-    annotations: testInfo.annotations,
+    annotations: testInfo.annotations.filter(a => !staticAnnotations.has(a)),
     timeout: testInfo.timeout,
   };
 }

packages/playwright/types/testReporter.d.ts

@@ -307,6 +307,7 @@ export interface JSONReportTestResult {
     body?: string;
     contentType: string;
   }[];
+  annotations: { type: string, description?: string }[];
   errorLocation?: Location;
 }
 
@@ -445,10 +446,8 @@ export interface TestCase {
    *   [test.skip([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip),
    *   [test.fixme([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme)
    *   and
-   *   [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail);
-   * - annotations appended to
-   *   [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations) during the test
-   *   execution.
+   *   [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail)
+   *   prior to test execution.
    *
    * Annotations are available during test execution through
    * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations).
@@ -592,6 +591,34 @@ export interface TestError {
  * A result of a single [TestCase](https://playwright.dev/docs/api/class-testcase) run.
  */
 export interface TestResult {
+  /**
+   * The list of annotations appended during test execution. Includes:
+   * - annotations implicitly added by methods
+   *   [test.skip([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip),
+   *   [test.fixme([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme)
+   *   and
+   *   [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail)
+   *   during test execution;
+   * - annotations appended to
+   *   [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations).
+   *
+   * Annotations are available during test execution through
+   * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations).
+   *
+   * Learn more about [test annotations](https://playwright.dev/docs/test-annotations).
+   */
+  annotations: Array<{
+    /**
+     * Annotation type, for example `'skip'` or `'fail'`.
+     */
+    type: string;
+
+    /**
+     * Optional description.
+     */
+    description?: string;
+  }>;
+
   /**
    * The list of files or buffers attached during the test execution through
    * [testInfo.attachments](https://playwright.dev/docs/api/class-testinfo#test-info-attachments).

packages/trace-viewer/src/ui/uiModeTraceView.tsx

@@ -86,6 +86,8 @@ export const TraceView: React.FC<{
     };
   }, [outputDir, item, setModel, counter, setCounter, pathSeparator]);
 
+  const annotations = item.testCase ? [...item.testCase.annotations, ...(item.testCase.results[0]?.annotations ?? [])] : [];
+
   return <Workbench
     key='workbench'
     model={model?.model}
@@ -94,7 +96,7 @@ export const TraceView: React.FC<{
     fallbackLocation={item.testFile}
     isLive={model?.isLive}
     status={item.treeItem?.status}
-    annotations={item.testCase?.annotations || []}
+    annotations={annotations}
     onOpenExternally={onOpenExternally}
     revealSource={revealSource}
   />;

tests/playwright-test/access-data.spec.ts

@@ -60,7 +60,7 @@ test('should access annotations in fixture', async ({ runInlineTest }) => {
   });
   expect(exitCode).toBe(0);
   const test = report.suites[0].specs[0].tests[0];
-  expect(test.annotations).toEqual([{ type: 'slow', description: 'just slow' }, { type: 'myname', description: 'hello' }]);
+  expect(test.results[0].annotations).toEqual([{ type: 'slow', description: 'just slow' }, { type: 'myname', description: 'hello' }]);
   expect(test.results[0].stdout).toEqual([{ text: 'console.log\n' }]);
   expect(test.results[0].stderr).toEqual([{ text: 'console.error\n' }]);
 });

tests/playwright-test/playwright-test-fixtures.ts

@@ -434,7 +434,7 @@ export function expectTestHelper(result: RunResult) {
     for (const test of tests) {
       expect(test.expectedStatus, `title: ${title}`).toBe(expectedStatus);
       expect(test.status, `title: ${title}`).toBe(status);
-      expect(test.annotations.map(a => a.type), `title: ${title}`).toEqual(annotations);
+      expect([...test.annotations, ...test.results.flatMap(r => r.annotations)].map(a => a.type), `title: ${title}`).toEqual(annotations);
     }
   };
 }

tests/playwright-test/reporter-html.spec.ts

@@ -829,6 +829,33 @@ for (const useIntermediateMergeReport of [true, false] as const) {
       await expect(page.locator('.test-case-annotation')).toHaveText('issue: I am not interested in this test');
     });
 
+    test('should render dynamic annotations at test result level', async ({ runInlineTest, page, showReport }) => {
+      const result = await runInlineTest({
+        'playwright.config.js': `
+          module.exports = { timeout: 1500, retries: 3 };
+        `,
+        'a.test.js': `
+          import { test, expect } from '@playwright/test';
+          test('annotated test', async ({}) => {
+            test.info().annotations.push({ type: 'foo', description: 'retry #' + test.info().retry });
+            test.info().annotations.push({ type: 'bar', description: 'static value' });
+            throw new Error('fail');
+          });
+        `,
+      }, { reporter: 'dot,html' }, { PLAYWRIGHT_HTML_OPEN: 'never' });
+      expect(result.failed).toBe(1);
+
+      await showReport();
+      await page.getByRole('link', { name: 'annotated test' }).click();
+      await page.getByRole('tab', { name: 'Retry #1' }).click();
+      await expect(page.getByTestId('test-case-annotations')).toContainText('foo: retry #1');
+
+      await page.getByRole('tab', { name: 'Retry #3' }).click();
+      await expect(page.getByTestId('test-case-annotations')).toContainText('foo: retry #3');
+
+      await expect(page.getByTestId('test-case-annotations').getByText('static value')).toHaveCount(1);
+    });
+
     test('should render annotations as link if needed', async ({ runInlineTest, page, showReport, server }) => {
       const result = await runInlineTest({
         'playwright.config.js': `
@@ -2431,12 +2458,13 @@ for (const useIntermediateMergeReport of [true, false] as const) {
         'a.test.js': `
           const { test, expect } = require('@playwright/test');
           test('annotated test',{ annotation :[{type:'key',description:'value'}]}, async ({}) => {expect(1).toBe(1);});
+          test('slow test', () => { test.slow(); });
           test('non-annotated test', async ({}) => {expect(1).toBe(2);});
         `,
       }, { reporter: 'dot,html' }, { PW_TEST_HTML_REPORT_OPEN: 'never' });
 
       expect(result.exitCode).toBe(1);
-      expect(result.passed).toBe(1);
+      expect(result.passed).toBe(2);
       expect(result.failed).toBe(1);
 
       await showReport();
@@ -2456,6 +2484,11 @@ for (const useIntermediateMergeReport of [true, false] as const) {
         await expect(page.getByText('non-annotated test')).not.toBeVisible();
         await expect(page.getByText('annotated test')).toBeVisible();
       });
+
+      await test.step('filter by result annotation', async () => {
+        await searchInput.fill('annot:slow');
+        await expect(page.getByText('slow test')).toBeVisible();
+      });
     });
 
     test('tests should filter by fileName:line/column', async ({ runInlineTest, showReport, page }) => {

tests/playwright-test/retry.spec.ts

@@ -260,5 +260,5 @@ test('failed and skipped on retry should be marked as flaky', async ({ runInline
   expect(result.failed).toBe(0);
   expect(result.flaky).toBe(1);
   expect(result.output).toContain('Failed on first run');
-  expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'Skipped on first retry' }]);
+  expect(result.report.suites[0].specs[0].tests[0].results[1].annotations).toEqual([{ type: 'skip', description: 'Skipped on first retry' }]);
 });

tests/playwright-test/test-modifiers.spec.ts

@@ -106,7 +106,7 @@ test('test modifiers should work', async ({ runInlineTest }) => {
     const test = spec.tests[0];
     expect(test.expectedStatus).toBe(expectedStatus);
     expect(test.results[0].status).toBe(status);
-    expect(test.annotations).toEqual(annotations);
+    expect([...test.annotations, ...test.results.flatMap(r => r.annotations)]).toEqual(annotations);
   };
   expectTest('passed1', 'passed', 'passed', []);
   expectTest('passed2', 'passed', 'passed', []);
@@ -306,23 +306,6 @@ test.describe('test modifier annotations', () => {
     expectTest('focused fail by suite', 'failed', 'expected', ['fail']);
   });
 
-  test('should not multiple on retry', async ({ runInlineTest }) => {
-    const result = await runInlineTest({
-      'a.test.ts': `
-        import { test, expect } from '@playwright/test';
-        test('retry', () => {
-          test.info().annotations.push({ type: 'example' });
-          expect(1).toBe(2);
-        });
-      `,
-    }, { retries: 3 });
-    const expectTest = expectTestHelper(result);
-
-    expect(result.exitCode).toBe(1);
-    expect(result.passed).toBe(0);
-    expectTest('retry', 'passed', 'unexpected', ['example']);
-  });
-
   test('should not multiply on repeat-each', async ({ runInlineTest }) => {
     const result = await runInlineTest({
       'a.test.ts': `
@@ -424,7 +407,7 @@ test('should skip inside fixture', async ({ runInlineTest }) => {
   });
   expect(result.exitCode).toBe(0);
   expect(result.skipped).toBe(1);
-  expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
+  expect(result.report.suites[0].specs[0].tests[0].results[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
 });
 
 test('modifier with a function should throw in the test', async ({ runInlineTest }) => {
@@ -477,8 +460,8 @@ test('test.skip with worker fixtures only should skip before hooks and tests', a
   expect(result.passed).toBe(1);
   expect(result.skipped).toBe(2);
   expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([]);
-  expect(result.report.suites[0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
-  expect(result.report.suites[0].suites![0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
+  expect(result.report.suites[0].suites![0].specs[0].tests[0].results[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
+  expect(result.report.suites[0].suites![0].suites![0].specs[0].tests[0].results[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
   expect(result.outputLines).toEqual([
     'beforeEach',
     'passed',
@@ -615,8 +598,8 @@ test('should skip all tests from beforeAll', async ({ runInlineTest }) => {
     'beforeAll',
     'afterAll',
   ]);
-  expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
-  expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
+  expect(result.report.suites[0].specs[0].tests[0].results[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
+  expect(result.report.suites[0].specs[1].tests[0].results[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]);
 });
 
 test('should report skipped tests in-order with correct properties', async ({ runInlineTest }) => {
@@ -712,7 +695,7 @@ test('static modifiers should be added in serial mode', async ({ runInlineTest }
   expect(result.passed).toBe(0);
   expect(result.skipped).toBe(2);
   expect(result.didNotRun).toBe(1);
-  expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'slow' }]);
+  expect(result.report.suites[0].specs[0].tests[0].results[0].annotations).toEqual([{ type: 'slow' }]);
   expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'fixme' }]);
   expect(result.report.suites[0].specs[2].tests[0].annotations).toEqual([{ type: 'skip' }]);
   expect(result.report.suites[0].specs[3].tests[0].annotations).toEqual([]);

utils/generate_types/overrides-testReporter.d.ts

@@ -125,6 +125,7 @@ export interface JSONReportTestResult {
     body?: string;
     contentType: string;
   }[];
+  annotations: { type: string, description?: string }[];
   errorLocation?: Location;
 }