chore: make find-related-test-files work through plugins (#32465)

Also switches it to the task runner.

packages/playwright-ct-core/index.js

@@ -16,7 +16,7 @@
 
 const { test: baseTest, expect, devices, defineConfig: originalDefineConfig } = require('playwright/test');
 const { fixtures } = require('./lib/mount');
-const { clearCacheCommand, findRelatedTestFilesCommand } = require('./lib/cliOverrides');
+const { clearCacheCommand } = require('./lib/cliOverrides');
 const { createPlugin } = require('./lib/vitePlugin');
 
 const defineConfig = (...configs) => {
@@ -31,7 +31,6 @@ const defineConfig = (...configs) => {
       ],
       cli: {
         'clear-cache': clearCacheCommand,
-        'find-related-test-files': findRelatedTestFilesCommand,
       },
     }
   };

packages/playwright-ct-core/src/cliOverrides.ts

@@ -15,8 +15,7 @@
  * limitations under the License.
  */
 
-import { affectedTestFiles, cacheDir } from 'playwright/lib/transform/compilationCache';
-import { buildBundle } from './vitePlugin';
+import { cacheDir } from 'playwright/lib/transform/compilationCache';
 import { resolveDirs } from './viteUtils';
 import type { FullConfigInternal } from 'playwright/lib/common/config';
 import { removeFolderAndLogToConsole } from 'playwright/lib/runner/testServer';
@@ -27,8 +26,3 @@ export async function clearCacheCommand(config: FullConfigInternal) {
     await removeFolderAndLogToConsole(dirs.outDir);
   await removeFolderAndLogToConsole(cacheDir);
 }
-
-export async function findRelatedTestFilesCommand(files: string[],  config: FullConfigInternal) {
-  await buildBundle(config.config, config.configDir);
-  return { testFiles: affectedTestFiles(files) };
-}

packages/playwright/src/program.ts

@@ -86,7 +86,8 @@ function addFindRelatedTestFilesCommand(program: Command) {
   command.description('Returns the list of related tests to the given files');
   command.option('-c, --config <file>', `Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}"`);
   command.action(async (files, options) => {
-    await withRunnerAndMutedWrite(options.config, runner => runner.findRelatedTestFiles('in-process', files));
+    const resolvedFiles = (files as string[]).map(file => path.resolve(process.cwd(), file));
+    await withRunnerAndMutedWrite(options.config, runner => runner.findRelatedTestFiles(resolvedFiles));
   });
 }
 

packages/playwright/src/runner/reporters.ts

@@ -86,12 +86,22 @@ export async function createReporterForTestServer(file: string, messageSink: (me
   }));
 }
 
-export function createConsoleReporter() {
-  return wrapReporterAsV2({
+interface ErrorCollectingReporter extends ReporterV2 {
+  errors(): TestError[];
+}
+
+export function createErrorCollectingReporter(writeToConsole?: boolean): ErrorCollectingReporter {
+  const errors: TestError[] = [];
+  const reporterV2 = wrapReporterAsV2({
     onError(error: TestError) {
-      process.stdout.write(formatError(error, colors.enabled).message + '\n');
+      errors.push(error);
+      if (writeToConsole)
+        process.stdout.write(formatError(error, colors.enabled).message + '\n');
     }
   });
+  const reporter = reporterV2 as ErrorCollectingReporter;
+  reporter.errors = () => errors;
+  return reporter;
 }
 
 function reporterOptions(config: FullConfigInternal, mode: 'list' | 'test' | 'merge', isTestServer: boolean) {

packages/playwright/src/runner/runner.ts

@@ -21,11 +21,9 @@ import { monotonicTime } from 'playwright-core/lib/utils';
 import type { FullResult, TestError } from '../../types/testReporter';
 import { webServerPluginsForConfig } from '../plugins/webServerPlugin';
 import { collectFilesForProject, filterProjects } from './projectUtils';
-import { createConsoleReporter, createReporters } from './reporters';
-import { TestRun, createTaskRunner, createTaskRunnerForDevServer, createTaskRunnerForList } from './tasks';
+import { createErrorCollectingReporter, createReporters } from './reporters';
+import { TestRun, createTaskRunner, createTaskRunnerForDevServer, createTaskRunnerForList, createTaskRunnerForRelatedTestFiles } from './tasks';
 import type { FullConfigInternal } from '../common/config';
-import type { Suite } from '../common/test';
-import { wrapReporterAsV2 } from '../reporters/reporterV2';
 import { affectedTestFiles } from '../transform/compilationCache';
 import { InternalReporter } from '../reporters/internalReporter';
 
@@ -109,43 +107,22 @@ export class Runner {
     return status;
   }
 
-  async loadAllTests(mode: 'in-process' | 'out-of-process' = 'in-process'): Promise<{ status: FullResult['status'], suite?: Suite, errors: TestError[] }> {
-    const config = this._config;
-    const errors: TestError[] = [];
-    const reporter = new InternalReporter([wrapReporterAsV2({
-      onError(error: TestError) {
-        errors.push(error);
-      }
-    })]);
-    const taskRunner = createTaskRunnerForList(config, reporter, mode, { failOnLoadErrors: true });
-    const testRun = new TestRun(config);
-    reporter.onConfigure(config.config);
-
-    const taskStatus = await taskRunner.run(testRun, 0);
-    let status: FullResult['status'] = testRun.failureTracker.result();
-    if (status === 'passed' && taskStatus !== 'passed')
-      status = taskStatus;
-    const modifiedResult = await reporter.onEnd({ status });
-    if (modifiedResult && modifiedResult.status)
-      status = modifiedResult.status;
+  async findRelatedTestFiles(files: string[]): Promise<FindRelatedTestFilesReport>  {
+    const errorReporter = createErrorCollectingReporter();
+    const reporter = new InternalReporter([errorReporter]);
+    const taskRunner = createTaskRunnerForRelatedTestFiles(this._config, reporter, 'in-process', true);
+    const testRun = new TestRun(this._config);
+    reporter.onConfigure(this._config.config);
+    const status = await taskRunner.run(testRun, 0);
+    await reporter.onEnd({ status });
     await reporter.onExit();
-    return { status, suite: testRun.rootSuite, errors };
-  }
-
-  async findRelatedTestFiles(mode: 'in-process' | 'out-of-process', files: string[]): Promise<FindRelatedTestFilesReport>  {
-    const result = await this.loadAllTests(mode);
-    if (result.status !== 'passed' || !result.suite)
-      return { errors: result.errors, testFiles: [] };
-
-    const resolvedFiles = (files as string[]).map(file => path.resolve(process.cwd(), file));
-    const override = (this._config.config as any)['@playwright/test']?.['cli']?.['find-related-test-files'];
-    if (override)
-      return await override(resolvedFiles, this._config);
-    return { testFiles: affectedTestFiles(resolvedFiles) };
+    if (status !== 'passed')
+      return { errors: errorReporter.errors(), testFiles: [] };
+    return { testFiles: affectedTestFiles(files) };
   }
 
   async runDevServer() {
-    const reporter = new InternalReporter([createConsoleReporter()]);
+    const reporter = new InternalReporter([createErrorCollectingReporter(true)]);
     const taskRunner = createTaskRunnerForDevServer(this._config, reporter, 'in-process', true);
     const testRun = new TestRun(this._config);
     reporter.onConfigure(this._config.config);

packages/playwright/src/runner/tasks.ts

@@ -129,6 +129,16 @@ export function createTaskRunnerForDevServer(config: FullConfigInternal, reporte
   return taskRunner;
 }
 
+export function createTaskRunnerForRelatedTestFiles(config: FullConfigInternal, reporter: InternalReporter, mode: 'in-process' | 'out-of-process', setupPlugins: boolean): TaskRunner<TestRun> {
+  const taskRunner = TaskRunner.create<TestRun>(reporter, config.config.globalTimeout);
+  if (setupPlugins) {
+    for (const plugin of config.plugins)
+      taskRunner.addTask('plugin setup', createPluginSetupTask(plugin));
+  }
+  taskRunner.addTask('load tests', createLoadTask(mode, { failOnLoadErrors: true, filterOnly: false, populateDependencies: true }));
+  return taskRunner;
+}
+
 function createReportBeginTask(): Task<TestRun> {
   return {
     setup: async (reporter, { rootSuite }) => {
@@ -231,16 +241,19 @@ function createListFilesTask(): Task<TestRun> {
   };
 }
 
-function createLoadTask(mode: 'out-of-process' | 'in-process', options: { filterOnly: boolean, failOnLoadErrors: boolean, doNotRunDepsOutsideProjectFilter?: boolean }): Task<TestRun> {
+function createLoadTask(mode: 'out-of-process' | 'in-process', options: { filterOnly: boolean, failOnLoadErrors: boolean, doNotRunDepsOutsideProjectFilter?: boolean, populateDependencies?: boolean }): Task<TestRun> {
   return {
     setup: async (reporter, testRun, errors, softErrors) => {
       await collectProjectsAndTestFiles(testRun, !!options.doNotRunDepsOutsideProjectFilter);
       await loadFileSuites(testRun, mode, options.failOnLoadErrors ? errors : softErrors);
 
-      let cliOnlyChangedMatcher: Matcher | undefined = undefined;
-      if (testRun.config.cliOnlyChanged) {
+      if (testRun.config.cliOnlyChanged || options.populateDependencies) {
         for (const plugin of testRun.config.plugins)
           await plugin.instance?.populateDependencies?.();
+      }
+
+      let cliOnlyChangedMatcher: Matcher | undefined = undefined;
+      if (testRun.config.cliOnlyChanged) {
         const changedFiles = await detectChangedTestFiles(testRun.config.cliOnlyChanged, testRun.config.configDir);
         cliOnlyChangedMatcher = file => changedFiles.has(file);
       }

packages/playwright/src/runner/testServer.ts

@@ -20,16 +20,15 @@ import { installRootRedirect, openTraceInBrowser, openTraceViewerApp, registry,
 import { ManualPromise, gracefullyProcessExitDoNotHang, isUnderTest } from 'playwright-core/lib/utils';
 import type { Transport, HttpServer } from 'playwright-core/lib/utils';
 import type * as reporterTypes from '../../types/testReporter';
-import { collectAffectedTestFiles, dependenciesForTestFile } from '../transform/compilationCache';
+import { affectedTestFiles, collectAffectedTestFiles, dependenciesForTestFile } from '../transform/compilationCache';
 import type { ConfigLocation, FullConfigInternal } from '../common/config';
-import { createReporterForTestServer, createReporters } from './reporters';
-import { TestRun, createTaskRunnerForList, createTaskRunnerForTestServer, createTaskRunnerForWatchSetup, createTaskRunnerForListFiles, createTaskRunnerForDevServer } from './tasks';
+import { createErrorCollectingReporter, createReporterForTestServer, createReporters } from './reporters';
+import { TestRun, createTaskRunnerForList, createTaskRunnerForTestServer, createTaskRunnerForWatchSetup, createTaskRunnerForListFiles, createTaskRunnerForDevServer, createTaskRunnerForRelatedTestFiles } from './tasks';
 import { open } from 'playwright-core/lib/utilsBundle';
 import ListReporter from '../reporters/list';
 import { SigIntWatcher } from './sigIntWatcher';
 import { Watcher } from '../fsWatcher';
 import type { ReportEntry, TestServerInterface, TestServerInterfaceEventEmitters } from '../isomorphic/testServerInterface';
-import { Runner } from './runner';
 import type { ConfigCLIOverrides } from '../common/ipc';
 import { loadConfig, resolveConfigLocation, restartWithExperimentalTsEsm } from '../common/configLoader';
 import { webServerPluginsForConfig } from '../plugins/webServerPlugin';
@@ -362,11 +361,21 @@ export class TestServerDispatcher implements TestServerInterface {
   }
 
   async findRelatedTestFiles(params: Parameters<TestServerInterface['findRelatedTestFiles']>[0]): ReturnType<TestServerInterface['findRelatedTestFiles']> {
-    const { config, error } = await this._loadConfig();
-    if (error)
-      return { testFiles: [], errors: [error] };
-    const runner = new Runner(config!);
-    return runner.findRelatedTestFiles('out-of-process', params.files);
+    const errorReporter = createErrorCollectingReporter();
+    const reporter = new InternalReporter([errorReporter]);
+    const config = await this._loadConfigOrReportError(reporter);
+    if (!config)
+      return { errors: errorReporter.errors(), testFiles: [] };
+
+    const taskRunner = createTaskRunnerForRelatedTestFiles(config, reporter, 'out-of-process', false);
+    const testRun = new TestRun(config);
+    reporter.onConfigure(config.config);
+    const status = await taskRunner.run(testRun, 0);
+    await reporter.onEnd({ status });
+    await reporter.onExit();
+    if (status !== 'passed')
+      return { errors: errorReporter.errors(), testFiles: [] };
+    return { testFiles: affectedTestFiles(params.files) };
   }
 
   async stopTests() {

tests/playwright-test/find-related-tests.spec.ts

@@ -15,9 +15,6 @@
  */
 
 import { test, expect } from './playwright-test-fixtures';
-import path from 'path';
-
-export const ctReactCliEntrypoint = path.join(__dirname, '../../packages/playwright-ct-react/cli.js');
 
 test('should list related tests', async ({ runCLICommand }) => {
   const result = await runCLICommand({
@@ -77,7 +74,7 @@ test('should list related tests for ct', async ({ runCLICommand }) => {
         await mount(<Button />);
       });
     `,
-  }, 'find-related-test-files', ['helper.tsx'], ctReactCliEntrypoint);
+  }, 'find-related-test-files', ['helper.tsx']);
   expect(result.exitCode).toBe(0);
   const data = JSON.parse(result.stdout);
   expect(data).toEqual({
@@ -86,3 +83,18 @@ test('should list related tests for ct', async ({ runCLICommand }) => {
     ]
   });
 });
+
+test('should return errors', async ({ runCLICommand }) => {
+  const result = await runCLICommand({
+    'a.spec.ts': `
+      const a = 1;
+      const a = 2;
+    `,
+  }, 'find-related-test-files', ['a.spec.ts']);
+  expect(result.exitCode).toBe(0);
+  const data = JSON.parse(result.stdout);
+  expect(data).toEqual({ testFiles: [], errors: [
+    expect.objectContaining({ message: expect.stringContaining(`Identifier 'a' has already been declared`) }),
+    expect.objectContaining({ message: expect.stringContaining(`No tests found`) }),
+  ] });
+});

tests/playwright-test/test-server.spec.ts

@@ -18,7 +18,7 @@ import { test as baseTest, expect } from './ui-mode-fixtures';
 import { TestServerConnection } from '../../packages/playwright/lib/isomorphic/testServerConnection';
 import { playwrightCtConfigText } from './playwright-test-fixtures';
 import ws from 'ws';
-import type { TestChildProcess } from 'tests/config/commonFixtures';
+import type { TestChildProcess } from '../config/commonFixtures';
 
 class WSTransport {
   private _ws: ws.WebSocket;
@@ -70,6 +70,24 @@ const test = baseTest.extend<{ startTestServer: () => Promise<TestServerConnecti
   }
 });
 
+const ctFiles = {
+  'playwright.config.ts': playwrightCtConfigText,
+  'playwright/index.html': `<script type="module" src="./index.ts"></script>`,
+  'playwright/index.ts': ``,
+  'src/button.tsx': `
+    export const Button = () => <button>Button</button>;
+  `,
+  'src/button.test.tsx': `
+    import { test, expect } from '@playwright/experimental-ct-react';
+    import { Button } from './button';
+
+    test('pass', async ({ mount }) => {
+      const component = await mount(<Button></Button>);
+      await expect(component).toHaveText('Button', { timeout: 1 });
+    });
+  `,
+};
+
 test('file watching', async ({ startTestServer, writeFiles }, testInfo) => {
   await writeFiles({
     'utils.ts': `
@@ -125,23 +143,7 @@ test('stdio interception', async ({ startTestServer, writeFiles }) => {
 });
 
 test('start dev server', async ({ startTestServer, writeFiles, runInlineTest }) => {
-  await writeFiles({
-    'playwright.config.ts': playwrightCtConfigText,
-    'playwright/index.html': `<script type="module" src="./index.ts"></script>`,
-    'playwright/index.ts': ``,
-    'src/button.tsx': `
-      export const Button = () => <button>Button</button>;
-    `,
-    'src/button.test.tsx': `
-      import { test, expect } from '@playwright/experimental-ct-react';
-      import { Button } from './button';
-
-      test('pass', async ({ mount }) => {
-        const component = await mount(<Button></Button>);
-        await expect(component).toHaveText('Button', { timeout: 1 });
-      });
-    `,
-  });
+  await writeFiles(ctFiles);
 
   const testServerConnection = await startTestServer();
   await testServerConnection.initialize({ interceptStdio: true });
@@ -156,3 +158,38 @@ test('start dev server', async ({ startTestServer, writeFiles, runInlineTest })
   expect((await testServerConnection.stopDevServer({})).status).toBe('passed');
   expect((await testServerConnection.runGlobalTeardown({})).status).toBe('passed');
 });
+
+test('find related test files errors', async ({ startTestServer, writeFiles }) => {
+  await writeFiles({
+    'a.spec.ts': `
+      const a = 1;
+      const a = 2;
+    `,
+  });
+  const testServerConnection = await startTestServer();
+  await testServerConnection.initialize({ interceptStdio: true });
+  expect((await testServerConnection.runGlobalSetup({})).status).toBe('passed');
+
+  const aSpecTs = test.info().outputPath('a.spec.ts');
+  const result = await testServerConnection.findRelatedTestFiles({ files: [aSpecTs] });
+  expect(result).toEqual({ testFiles: [], errors: [
+    expect.objectContaining({ message: expect.stringContaining(`Identifier 'a' has already been declared`) }),
+    expect.objectContaining({ message: expect.stringContaining(`No tests found`) }),
+  ] });
+
+  expect((await testServerConnection.runGlobalTeardown({})).status).toBe('passed');
+});
+
+test('find related test files', async ({ startTestServer, writeFiles }) => {
+  await writeFiles(ctFiles);
+  const testServerConnection = await startTestServer();
+  await testServerConnection.initialize({ interceptStdio: true });
+  expect((await testServerConnection.runGlobalSetup({})).status).toBe('passed');
+
+  const buttonTsx = test.info().outputPath('src/button.tsx');
+  const buttonTestTsx = test.info().outputPath('src/button.test.tsx');
+  const result = await testServerConnection.findRelatedTestFiles({ files: [buttonTsx] });
+  expect(result).toEqual({ testFiles: [buttonTestTsx] });
+
+  expect((await testServerConnection.runGlobalTeardown({})).status).toBe('passed');
+});