feat(onEnd): allow overriding the exit code (#27010)

Fixes: https://github.com/microsoft/playwright/issues/26858

docs/src/test-reporter-api/class-reporter.md

@@ -106,8 +106,11 @@ The root suite that contains all projects, files and test cases.
 
 ## optional async method: Reporter.onEnd
 * since: v1.10
+- `result` ?<[Object]>
+  - `status` ?<[FullStatus]<"passed"|"failed"|"timedout"|"interrupted">>
 
 Called after all tests have been run, or testing has been interrupted. Note that this method may return a [Promise] and Playwright Test will await it.
+Reporter is allowed to override the status and hence affect the exit code of the test runner.
 
 ### param: Reporter.onEnd.result
 * since: v1.10

packages/playwright/src/isomorphic/teleReceiver.ts

@@ -306,8 +306,8 @@ export class TeleReporterReceiver {
     }
   }
 
-  private _onEnd(result: JsonFullResult): Promise<void> | void {
+  private async _onEnd(result: JsonFullResult): Promise<void> {
-    return this._reporter.onEnd?.({
+    await this._reporter.onEnd?.({
       status: result.status,
       startTime: new Date(result.startTime),
       duration: result.duration,

packages/playwright/src/reporters/internalReporter.ts

@@ -72,7 +72,7 @@ export class InternalReporter {
       // onBegin was not reported, emit it.
       this.onBegin(new Suite('', 'root'));
     }
-    await this._reporter.onEnd({
+    return await this._reporter.onEnd({
       ...result,
       startTime: this._startTime!,
       duration: monotonicTime() - this._monotonicStartTime!,

packages/playwright/src/reporters/multiplexer.ts

@@ -60,8 +60,12 @@ export class Multiplexer implements ReporterV2 {
   }
 
   async onEnd(result: FullResult) {
-    for (const reporter of this._reporters)
+    for (const reporter of this._reporters) {
-      await wrapAsync(() => reporter.onEnd(result));
+      const outResult = await wrapAsync(() => reporter.onEnd(result));
+      if (outResult?.status)
+        result.status = outResult.status;
+    }
+    return result;
   }
 
   async onExit() {
@@ -93,9 +97,9 @@ export class Multiplexer implements ReporterV2 {
   }
 }
 
-async function wrapAsync(callback: () => void | Promise<void>) {
+async function wrapAsync<T>(callback: () => T | Promise<T>) {
   try {
-    await callback();
+    return await callback();
   } catch (e) {
     console.error('Error in reporter', e);
   }

packages/playwright/src/reporters/reporterV2.ts

@@ -23,7 +23,7 @@ export interface ReporterV2 {
   onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
   onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult): void;
   onTestEnd(test: TestCase, result: TestResult): void;
-  onEnd(result: FullResult): void | Promise<void>;
+  onEnd(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void;
   onExit(): void | Promise<void>;
   onError(error: TestError): void;
   onStepBegin(test: TestCase, result: TestResult, step: TestStep): void;
@@ -104,7 +104,7 @@ class ReporterV2Wrapper implements ReporterV2 {
   }
 
   async onEnd(result: FullResult) {
-    await this._reporter.onEnd?.(result);
+    return await this._reporter.onEnd?.(result);
   }
 
   async onExit() {

packages/playwright/src/runner/runner.ts

@@ -90,7 +90,10 @@ export class Runner {
     let status: FullResult['status'] = testRun.failureTracker.result();
     if (status === 'passed' && taskStatus !== 'passed')
       status = taskStatus;
-    await reporter.onEnd({ status });
+    const modifiedResult = await reporter.onEnd({ status });
+    if (modifiedResult && modifiedResult.status)
+      status = modifiedResult.status;
+
     await reporter.onExit();
 
     // Calling process.exit() might truncate large stdout/stderr output.

packages/playwright/types/testReporter.d.ts

@@ -410,7 +410,8 @@ export interface Reporter {
   onBegin?(config: FullConfig, suite: Suite): void;
   /**
    * Called after all tests have been run, or testing has been interrupted. Note that this method may return a [Promise]
-   * and Playwright Test will await it.
+   * and Playwright Test will await it. Reporter is allowed to override the status and hence affect the exit code of the
+   * test runner.
    * @param result Result of the full test run, `status` can be one of:
    * - `'passed'` - Everything went as expected.
    * - `'failed'` - Any test has failed.
@@ -419,7 +420,7 @@ export interface Reporter {
    * been reached.
    * - `'interrupted'` - Interrupted by the user.
    */
-  onEnd?(result: FullResult): void | Promise<void>;
+  onEnd?(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void;
   /**
    * Called on some global error, for example unhandled exception in the worker process.
    * @param error The error.

tests/playwright-test/reporter-onend.spec.ts

@@ -0,0 +1,40 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+const reporter = `
+class Reporter {
+  async onEnd() {
+    return { status: 'passed' };
+  }
+}
+module.exports = Reporter;
+`;
+
+test('should override exit code', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'reporter.ts': reporter,
+    'playwright.config.ts': `module.exports = { reporter: './reporter' };`,
+    'a.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('fail', async ({}) => {
+        expect(1 + 1).toBe(3);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});

utils/generate_types/overrides-testReporter.d.ts

@@ -55,7 +55,7 @@ export interface FullResult {
 
 export interface Reporter {
   onBegin?(config: FullConfig, suite: Suite): void;
-  onEnd?(result: FullResult): void | Promise<void>;
+  onEnd?(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void;
 }
 
 export interface JSONReport {