feat(config): failOnFlakyTests option (#35109)

docs/src/test-api/class-fullconfig.md

@@ -10,6 +10,12 @@ Resolved configuration which is accessible via [`property: TestInfo.config`] and
 
 Path to the configuration file used to run the tests. The value is an empty string if no config file was used.
 
+## property: FullConfig.failOnFlakyTests
+* since: v1.52
+- type: <[boolean]>
+
+See [`property: TestConfig.failOnFlakyTests`].
+
 ## property: FullConfig.forbidOnly
 * since: v1.10
 - type: <[boolean]>

docs/src/test-api/class-testconfig.md

@@ -108,6 +108,24 @@ export default defineConfig({
 });
 ```
 
+## property: TestConfig.failOnFlakyTests
+* since: v1.52
+- type: ?<[boolean]>
+
+Whether to exit with an error if any tests are marked as flaky. Useful on CI.
+
+Also available in the [command line](../test-cli.md) with the `--fail-on-flaky-tests` option.
+
+**Usage**
+
+```js title="playwright.config.ts"
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  failOnFlakyTests: !!process.env.CI,
+});
+```
+
 ## property: TestConfig.forbidOnly
 * since: v1.10
 - type: ?<[boolean]>

packages/playwright/src/common/config.ts

@@ -56,7 +56,6 @@ export class FullConfigInternal {
   cliProjectFilter?: string[];
   cliListOnly = false;
   cliPassWithNoTests?: boolean;
-  cliFailOnFlakyTests?: boolean;
   cliLastFailed?: boolean;
   testIdMatcher?: Matcher;
   lastFailedTestIdMatcher?: Matcher;
@@ -90,6 +89,7 @@ export class FullConfigInternal {
     this.config = {
       configFile: resolvedConfigFile,
       rootDir: pathResolve(configDir, userConfig.testDir) || configDir,
+      failOnFlakyTests: takeFirst(configCLIOverrides.failOnFlakyTests, userConfig.failOnFlakyTests, false),
       forbidOnly: takeFirst(configCLIOverrides.forbidOnly, userConfig.forbidOnly, false),
       fullyParallel: takeFirst(configCLIOverrides.fullyParallel, userConfig.fullyParallel, false),
       globalSetup: this.globalSetups[0] ?? null,

packages/playwright/src/common/ipc.ts

@@ -25,6 +25,7 @@ import type { SerializedCompilationCache  } from '../transform/compilationCache'
 
 export type ConfigCLIOverrides = {
   debug?: boolean;
+  failOnFlakyTests?: boolean;
   forbidOnly?: boolean;
   fullyParallel?: boolean;
   globalTimeout?: number;

packages/playwright/src/isomorphic/teleReceiver.ts

@@ -593,6 +593,7 @@ export class TeleTestResult implements reporterTypes.TestResult {
 export type TeleFullProject = reporterTypes.FullProject;
 
 export const baseFullConfig: reporterTypes.FullConfig = {
+  failOnFlakyTests: false,
   forbidOnly: false,
   fullyParallel: false,
   globalSetup: null,

packages/playwright/src/program.ts

@@ -172,7 +172,6 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
   config.cliListOnly = !!opts.list;
   config.cliProjectFilter = opts.project || undefined;
   config.cliPassWithNoTests = !!opts.passWithNoTests;
-  config.cliFailOnFlakyTests = !!opts.failOnFlakyTests;
   config.cliLastFailed = !!opts.lastFailed;
 
   // Evaluate project filters against config before starting execution. This enables a consistent error message across run modes
@@ -294,6 +293,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid
     updateSnapshots = 'updateSnapshots' in options ? 'changed' : undefined;
 
   const overrides: ConfigCLIOverrides = {
+    failOnFlakyTests: options.failOnFlakyTests ? true : undefined,
     forbidOnly: options.forbidOnly ? true : undefined,
     fullyParallel: options.fullyParallel ? true : undefined,
     globalTimeout: options.globalTimeout ? parseInt(options.globalTimeout, 10) : undefined,

packages/playwright/src/runner/failureTracker.ts

@@ -49,7 +49,7 @@ export class FailureTracker {
   }
 
   result(): 'failed' | 'passed' {
-    return this._hasWorkerErrors || this.hasReachedMaxFailures() || this.hasFailedTests() || (this._config.cliFailOnFlakyTests && this.hasFlakyTests()) ? 'failed' : 'passed';
+    return this._hasWorkerErrors || this.hasReachedMaxFailures() || this.hasFailedTests() || (this._config.config.failOnFlakyTests && this.hasFlakyTests()) ? 'failed' : 'passed';
   }
 
   hasFailedTests() {
@@ -63,4 +63,5 @@ export class FailureTracker {
   maxFailures() {
     return this._config.config.maxFailures;
   }
+
 }

packages/playwright/types/test.d.ts

@@ -1177,6 +1177,25 @@ interface TestConfig<TestArgs = {}, WorkerArgs = {}> {
     };
   };
 
+  /**
+   * Whether to exit with an error if any tests are marked as flaky. Useful on CI.
+   *
+   * Also available in the [command line](https://playwright.dev/docs/test-cli) with the `--fail-on-flaky-tests` option.
+   *
+   * **Usage**
+   *
+   * ```js
+   * // playwright.config.ts
+   * import { defineConfig } from '@playwright/test';
+   *
+   * export default defineConfig({
+   *   failOnFlakyTests: !!process.env.CI,
+   * });
+   * ```
+   *
+   */
+  failOnFlakyTests?: boolean;
+
   /**
    * Whether to exit with an error if any tests or groups are marked as
    * [test.only(title[, details, body])](https://playwright.dev/docs/api/class-test#test-only) or
@@ -1911,6 +1930,12 @@ export interface FullConfig<TestArgs = {}, WorkerArgs = {}> {
    */
   configFile?: string;
 
+  /**
+   * See
+   * [testConfig.failOnFlakyTests](https://playwright.dev/docs/api/class-testconfig#test-config-fail-on-flaky-tests).
+   */
+  failOnFlakyTests: boolean;
+
   /**
    * See [testConfig.forbidOnly](https://playwright.dev/docs/api/class-testconfig#test-config-forbid-only).
    */

tests/playwright-test/config.spec.ts

@@ -72,6 +72,25 @@ test('should prioritize command line timeout over project timeout', async ({ run
   expect(result.output).toContain('Test timeout of 500ms exceeded.');
 });
 
+test('should support failOnFlakyTests config option', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+        module.exports = {
+          failOnFlakyTests: true,
+          retries: 1
+        };
+    `,
+    'a.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('flake', async ({}, testInfo) => {
+        expect(testInfo.retry).toBe(1);
+      });
+    `,
+  }, { 'retries': 1 });
+  expect(result.exitCode).not.toBe(0);
+  expect(result.flaky).toBe(1);
+});
+
 test('should read config from --config, resolve relative testDir', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     'my.config.ts': `