Add snapshot testing on e2e test failure (#24672)

We have a currently unreproducible flaky e2e test. This PR captures snapshots on e2e test failures so we can better debug flaky e2e tests that don't fail locally.

.circleci/config.yml

@@ -291,6 +291,8 @@ jobs:
       - run:
           name: Cleanup build regression folder
           command: rm -r ./build-regression
+      - store_artifacts:
+          path: ./tmp/screenshots
 
   yarn_lint_build:
     docker: *docker

packages/react-devtools-inline/__tests__/__e2e__/components.test.js

@@ -210,7 +210,6 @@ test.describe('Components', () => {
     let count = await getComponentSearchResultsCount();
     expect(count).toBe('1 | 4');
 
-    await focusComponentSearch();
     page.keyboard.insertText('Item');
     count = await getComponentSearchResultsCount();
     expect(count).toBe('1 | 3');

packages/react-devtools-inline/playwright.config.js

@@ -18,9 +18,10 @@ const config = {
     react_version: process.env.REACT_VERSION
       ? semver.coerce(process.env.REACT_VERSION).version
       : reactVersion,
+    trace: 'retain-on-failure',
   },
   // Some of our e2e tests can be flaky. Retry tests to make sure the error isn't transient
-  retries: 2,
+  retries: 3,
 };
 
 module.exports = config;

scripts/circleci/run_devtools_e2e_tests.js

@@ -9,6 +9,7 @@ const ROOT_PATH = join(__dirname, '..', '..');
 const reactVersion = process.argv[2];
 const inlinePackagePath = join(ROOT_PATH, 'packages', 'react-devtools-inline');
 const shellPackagePath = join(ROOT_PATH, 'packages', 'react-devtools-shell');
+const screenshotPath = join(ROOT_PATH, 'tmp', 'screenshots');
 
 let buildProcess = null;
 let serverProcess = null;
@@ -115,9 +116,11 @@ function runTestShell() {
 async function runEndToEndTests() {
   logBright('Running e2e tests');
   if (!reactVersion) {
-    testProcess = spawn('yarn', ['test:e2e'], {cwd: inlinePackagePath});
+    testProcess = spawn('yarn', ['test:e2e', `--output=${screenshotPath}`], {
+      cwd: inlinePackagePath,
+    });
   } else {
-    testProcess = spawn('yarn', ['test:e2e'], {
+    testProcess = spawn('yarn', ['test:e2e', `--output=${screenshotPath}`], {
       cwd: inlinePackagePath,
       env: {...process.env, REACT_VERSION: reactVersion},
     });