feat(circus): enable writing async test event handlers (#9397)

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 - `[babel-jest]` Support passing `supportsDynamicImport` and `supportsStaticESM` ([#9766](https://github.com/facebook/jest/pull/9766))
 - `[babel-preset-jest]` Enable all syntax plugins not enabled by default that works on current version of Node ([#9774](https://github.com/facebook/jest/pull/9774))
+- `[jest-circus]` Enable writing async test event handlers ([#9392](https://github.com/facebook/jest/pull/9392))
 - `[jest-runtime, @jest/transformer]` Support passing `supportsDynamicImport` and `supportsStaticESM` ([#9597](https://github.com/facebook/jest/pull/9597))
 
 ### Fixes

docs/Configuration.md

@@ -900,7 +900,7 @@ test('use jsdom in this test file', () => {
 
 You can create your own module that will be used for setting up the test environment. The module must export a class with `setup`, `teardown` and `runScript` methods. You can also pass variables from this module to your test suites by assigning them to `this.global` object – this will make them available in your test suites as global variables.
 
-The class may optionally expose a `handleTestEvent` method to bind to events fired by [`jest-circus`](https://github.com/facebook/jest/tree/master/packages/jest-circus).
+The class may optionally expose an asynchronous `handleTestEvent` method to bind to events fired by [`jest-circus`](https://github.com/facebook/jest/tree/master/packages/jest-circus). Normally, `jest-circus` test runner would pause until a promise returned from `handleTestEvent` gets fulfilled, **except for the next events**: `start_describe_definition`, `finish_describe_definition`, `add_hook`, `add_test` or `error` (for the up-to-date list you can look at [SyncEvent type in the types definitions](https://github.com/facebook/jest/tree/master/packages/jest-types/src/Circus.ts)). That is caused by backward compatibility reasons and `process.on('unhandledRejection', callback)` signature, but that usually should not be a problem for most of the use cases.
 
 Any docblock pragmas in test files will be passed to the environment constructor and can be used for per-test configuration. If the pragma does not have a value, it will be present in the object with it's value set to an empty string. If the pragma is not present, it will not be present in the object.
 
@@ -940,7 +940,7 @@ class CustomEnvironment extends NodeEnvironment {
     return super.runScript(script);
   }
 
-  handleTestEvent(event, state) {
+  async handleTestEvent(event, state) {
     if (event.name === 'test_start') {
       // ...
     }

e2e/__tests__/testEnvironmentCircusAsync.test.ts

@@ -0,0 +1,64 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {skipSuiteOnJasmine} from '@jest/test-utils';
+import runJest from '../runJest';
+
+skipSuiteOnJasmine();
+
+it('calls asynchronous handleTestEvent in testEnvironment', () => {
+  const result = runJest('test-environment-circus-async');
+  expect(result.failed).toEqual(true);
+
+  const lines = result.stdout.split('\n');
+  expect(lines).toMatchInlineSnapshot(`
+    Array [
+      "setup",
+      "warning: add_hook is a sync event",
+      "warning: start_describe_definition is a sync event",
+      "warning: add_hook is a sync event",
+      "warning: add_hook is a sync event",
+      "warning: add_test is a sync event",
+      "warning: add_test is a sync event",
+      "warning: finish_describe_definition is a sync event",
+      "add_hook",
+      "start_describe_definition",
+      "add_hook",
+      "add_hook",
+      "add_test",
+      "add_test",
+      "finish_describe_definition",
+      "run_start",
+      "run_describe_start",
+      "run_describe_start",
+      "test_start: passing test",
+      "hook_start: beforeEach",
+      "hook_success: beforeEach",
+      "hook_start: beforeEach",
+      "hook_success: beforeEach",
+      "test_fn_start: passing test",
+      "test_fn_success: passing test",
+      "hook_start: afterEach",
+      "hook_failure: afterEach",
+      "test_done: passing test",
+      "test_start: failing test",
+      "hook_start: beforeEach",
+      "hook_success: beforeEach",
+      "hook_start: beforeEach",
+      "hook_success: beforeEach",
+      "test_fn_start: failing test",
+      "test_fn_failure: failing test",
+      "hook_start: afterEach",
+      "hook_failure: afterEach",
+      "test_done: failing test",
+      "run_describe_finish",
+      "run_describe_finish",
+      "run_finish",
+      "teardown",
+    ]
+  `);
+});

e2e/test-environment-circus-async/CircusAsyncHandleTestEventEnvironment.js

@@ -0,0 +1,38 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+'use strict';
+
+const JSDOMEnvironment = require('jest-environment-jsdom');
+
+const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
+
+class TestEnvironment extends JSDOMEnvironment {
+  async handleTestEvent(event) {
+    await this.assertRunnerWaitsForHandleTestEvent(event);
+
+    if (event.hook) {
+      console.log(event.name + ': ' + event.hook.type);
+    } else if (event.test) {
+      console.log(event.name + ': ' + event.test.name);
+    } else {
+      console.log(event.name);
+    }
+  }
+
+  async assertRunnerWaitsForHandleTestEvent(event) {
+    if (this.pendingEvent) {
+      console.log(`warning: ${this.pendingEvent.name} is a sync event`);
+    }
+
+    this.pendingEvent = event;
+    await sleep(0);
+    this.pendingEvent = null;
+  }
+}
+
+module.exports = TestEnvironment;

e2e/test-environment-circus-async/__tests__/circusHandleTestEvent.test.js

@@ -0,0 +1,23 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @jest-environment ./CircusAsyncHandleTestEventEnvironment.js
+ */
+
+describe('suite', () => {
+  beforeEach(() => {});
+  afterEach(() => {
+    throw new Error();
+  });
+
+  test('passing test', () => {
+    expect(true).toBe(true);
+  });
+
+  test('failing test', () => {
+    expect(true).toBe(false);
+  });
+});

e2e/test-environment-circus-async/package.json

@@ -0,0 +1,5 @@
+{
+  "jest": {
+    "testEnvironment": "node"
+  }
+}

packages/jest-circus/README.md

@@ -18,7 +18,7 @@ import {Event, State} from 'jest-circus';
 class MyCustomEnvironment extends NodeEnvironment {
   //...
 
-  handleTestEvent(event: Event, state: State) {
+  async handleTestEvent(event: Event, state: State) {
     if (event.name === 'test_start') {
       // ...
     }
@@ -28,6 +28,8 @@ class MyCustomEnvironment extends NodeEnvironment {
 
 Mutating event or state data is currently unsupported and may cause unexpected behavior or break in a future release without warning. New events, event data, and/or state data will not be considered a breaking change and may be added in any minor release.
 
+Note, that `jest-circus` test runner would pause until a promise returned from `handleTestEvent` gets fulfilled. **However, there are a few events that do not conform to this rule, namely**: `start_describe_definition`, `finish_describe_definition`, `add_hook`, `add_test` or `error` (for the up-to-date list you can look at [SyncEvent type in the types definitions](https://github.com/facebook/jest/tree/master/packages/jest-types/src/Circus.ts)). That is caused by backward compatibility reasons and `process.on('unhandledRejection', callback)` signature, but that usually should not be a problem for most of the use cases.
+
 ## Installation
 
 Install `jest-circus` using yarn:

packages/jest-circus/src/eventHandler.ts

@@ -21,7 +21,11 @@ import {
   restoreGlobalErrorHandlers,
 } from './globalErrorHandlers';
 
-const eventHandler: Circus.EventHandler = (event, state): void => {
+// TODO: investigate why a shorter (event, state) signature results into TS7006 compiler error
+const eventHandler: Circus.EventHandler = (
+  event: Circus.Event,
+  state: Circus.State,
+): void => {
   switch (event.name) {
     case 'include_test_location_in_result': {
       state.includeTestLocationInResult = true;

packages/jest-circus/src/globalErrorHandlers.ts

@@ -6,11 +6,11 @@
  */
 
 import type {Circus} from '@jest/types';
-import {dispatch} from './state';
+import {dispatchSync} from './state';
 
 const uncaught: NodeJS.UncaughtExceptionListener &
   NodeJS.UnhandledRejectionListener = (error: unknown) => {
-  dispatch({error, name: 'error'});
+  dispatchSync({error, name: 'error'});
 };
 
 export const injectGlobalErrorHandlers = (

packages/jest-circus/src/index.ts

@@ -10,7 +10,7 @@ import {bind as bindEach} from 'jest-each';
 import {formatExecError} from 'jest-message-util';
 import {ErrorWithStack, isPromise} from 'jest-util';
 import type {Circus, Global} from '@jest/types';
-import {dispatch} from './state';
+import {dispatchSync} from './state';
 
 type THook = (fn: Circus.HookFn, timeout?: number) => void;
 type DescribeFn = (
@@ -52,7 +52,7 @@ const _dispatchDescribe = (
     asyncError.message = `Invalid second argument, ${blockFn}. It must be a callback function.`;
     throw asyncError;
   }
-  dispatch({
+  dispatchSync({
     asyncError,
     blockName,
     mode,
@@ -91,7 +91,7 @@ const _dispatchDescribe = (
     );
   }
 
-  dispatch({blockName, mode, name: 'finish_describe_definition'});
+  dispatchSync({blockName, mode, name: 'finish_describe_definition'});
 };
 
 const _addHook = (
@@ -109,7 +109,7 @@ const _addHook = (
     throw asyncError;
   }
 
-  dispatch({asyncError, fn, hookType, name: 'add_hook', timeout});
+  dispatchSync({asyncError, fn, hookType, name: 'add_hook', timeout});
 };
 
 // Hooks have to pass themselves to the HOF in order for us to trim stack traces.
@@ -179,7 +179,7 @@ const test: Global.It = (() => {
       throw asyncError;
     }
 
-    return dispatch({
+    return dispatchSync({
       asyncError,
       fn,
       mode,

packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapter.ts

@@ -38,7 +38,7 @@ const jestAdapter = async (
     config.prettierPath ? require(config.prettierPath) : null;
   const getBabelTraverse = () => require('@babel/traverse').default;
 
-  const {globals, snapshotState} = initialize({
+  const {globals, snapshotState} = await initialize({
     config,
     environment,
     getBabelTraverse,

packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapterInit.ts

@@ -36,7 +36,7 @@ type Process = NodeJS.Process;
 
 // TODO: hard to type
 // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
-export const initialize = ({
+export const initialize = async ({
   config,
   environment,
   getPrettier,
@@ -107,14 +107,14 @@ export const initialize = ({
     addEventHandler(environment.handleTestEvent.bind(environment));
   }
 
-  dispatch({
+  await dispatch({
     name: 'setup',
     parentProcess,
     testNamePattern: globalConfig.testNamePattern,
   });
 
   if (config.testLocationInResults) {
-    dispatch({
+    await dispatch({
       name: 'include_test_location_in_result',
     });
   }
@@ -220,7 +220,8 @@ export const runAndTransformResultsToJestFormat = async ({
         .join('\n');
   }
 
-  dispatch({name: 'teardown'});
+  await dispatch({name: 'teardown'});
+
   return {
     ...createEmptyTestResult(),
     console: undefined,
@@ -248,7 +249,7 @@ const handleSnapshotStateAfterRetry = (snapshotState: SnapshotStateType) => (
   }
 };
 
-const eventHandler = (event: Circus.Event) => {
+const eventHandler = async (event: Circus.Event) => {
   switch (event.name) {
     case 'test_start': {
       setState({currentTestName: getTestID(event.test)});

packages/jest-circus/src/run.ts

@@ -20,9 +20,9 @@ import {
 
 const run = async (): Promise<Circus.RunResult> => {
   const {rootDescribeBlock} = getState();
-  dispatch({name: 'run_start'});
+  await dispatch({name: 'run_start'});
   await _runTestsForDescribeBlock(rootDescribeBlock);
-  dispatch({name: 'run_finish'});
+  await dispatch({name: 'run_finish'});
   return makeRunResult(
     getState().rootDescribeBlock,
     getState().unhandledErrors,
@@ -32,7 +32,7 @@ const run = async (): Promise<Circus.RunResult> => {
 const _runTestsForDescribeBlock = async (
   describeBlock: Circus.DescribeBlock,
 ) => {
-  dispatch({describeBlock, name: 'run_describe_start'});
+  await dispatch({describeBlock, name: 'run_describe_start'});
   const {beforeAll, afterAll} = getAllHooksForDescribe(describeBlock);
 
   for (const hook of beforeAll) {
@@ -62,7 +62,7 @@ const _runTestsForDescribeBlock = async (
 
     while (numRetriesAvailable > 0 && test.errors.length > 0) {
       // Clear errors so retries occur
-      dispatch({name: 'test_retry', test});
+      await dispatch({name: 'test_retry', test});
 
       await _runTest(test);
       numRetriesAvailable--;
@@ -76,11 +76,12 @@ const _runTestsForDescribeBlock = async (
   for (const hook of afterAll) {
     await _callCircusHook({describeBlock, hook});
   }
-  dispatch({describeBlock, name: 'run_describe_finish'});
+
+  await dispatch({describeBlock, name: 'run_describe_finish'});
 };
 
 const _runTest = async (test: Circus.TestEntry): Promise<void> => {
-  dispatch({name: 'test_start', test});
+  await dispatch({name: 'test_start', test});
   const testContext = Object.create(null);
   const {hasFocusedTests, testNamePattern} = getState();
 
@@ -90,12 +91,12 @@ const _runTest = async (test: Circus.TestEntry): Promise<void> => {
     (testNamePattern && !testNamePattern.test(getTestID(test)));
 
   if (isSkipped) {
-    dispatch({name: 'test_skip', test});
+    await dispatch({name: 'test_skip', test});
     return;
   }
 
   if (test.mode === 'todo') {
-    dispatch({name: 'test_todo', test});
+    await dispatch({name: 'test_todo', test});
     return;
   }
 
@@ -119,10 +120,10 @@ const _runTest = async (test: Circus.TestEntry): Promise<void> => {
   // `afterAll` hooks should not affect test status (pass or fail), because if
   // we had a global `afterAll` hook it would block all existing tests until
   // this hook is executed. So we dispatch `test_done` right away.
-  dispatch({name: 'test_done', test});
+  await dispatch({name: 'test_done', test});
 };
 
-const _callCircusHook = ({
+const _callCircusHook = async ({
   hook,
   test,
   describeBlock,
@@ -132,32 +133,36 @@ const _callCircusHook = ({
   describeBlock?: Circus.DescribeBlock;
   test?: Circus.TestEntry;
   testContext?: Circus.TestContext;
-}): Promise<unknown> => {
-  dispatch({hook, name: 'hook_start'});
+}): Promise<void> => {
+  await dispatch({hook, name: 'hook_start'});
   const timeout = hook.timeout || getState().testTimeout;
-  return callAsyncCircusFn(hook.fn, testContext, {isHook: true, timeout})
-    .then(() => dispatch({describeBlock, hook, name: 'hook_success', test}))
-    .catch(error =>
-      dispatch({describeBlock, error, hook, name: 'hook_failure', test}),
-    );
+
+  try {
+    await callAsyncCircusFn(hook.fn, testContext, {isHook: true, timeout});
+    await dispatch({describeBlock, hook, name: 'hook_success', test});
+  } catch (error) {
+    await dispatch({describeBlock, error, hook, name: 'hook_failure', test});
+  }
 };
 
-const _callCircusTest = (
+const _callCircusTest = async (
   test: Circus.TestEntry,
   testContext: Circus.TestContext,
 ): Promise<void> => {
-  dispatch({name: 'test_fn_start', test});
+  await dispatch({name: 'test_fn_start', test});
   const timeout = test.timeout || getState().testTimeout;
   invariant(test.fn, `Tests with no 'fn' should have 'mode' set to 'skipped'`);
 
   if (test.errors.length) {
-    // We don't run the test if there's already an error in before hooks.
-    return Promise.resolve();
+    return; // We don't run the test if there's already an error in before hooks.
   }
 
-  return callAsyncCircusFn(test.fn, testContext, {isHook: false, timeout})
-    .then(() => dispatch({name: 'test_fn_success', test}))
-    .catch(error => dispatch({error, name: 'test_fn_failure', test}));
+  try {
+    await callAsyncCircusFn(test.fn, testContext, {isHook: false, timeout});
+    await dispatch({name: 'test_fn_success', test});
+  } catch (error) {
+    await dispatch({error, name: 'test_fn_failure', test});
+  }
 };
 
 export default run;

packages/jest-circus/src/state.ts

@@ -39,7 +39,13 @@ export const getState = (): Circus.State => global[STATE_SYM];
 export const setState = (state: Circus.State): Circus.State =>
   (global[STATE_SYM] = state);
 
-export const dispatch = (event: Circus.Event): void => {
+export const dispatch = async (event: Circus.AsyncEvent): Promise<void> => {
+  for (const handler of eventHandlers) {
+    await handler(event, getState());
+  }
+};
+
+export const dispatchSync = (event: Circus.SyncEvent): void => {
   for (const handler of eventHandlers) {
     handler(event, getState());
   }

packages/jest-environment/src/index.ts

@@ -50,7 +50,10 @@ export declare class JestEnvironment {
   getVmContext?(): Context | null;
   setup(): Promise<void>;
   teardown(): Promise<void>;
-  handleTestEvent?(event: Circus.Event, state: Circus.State): void;
+  handleTestEvent?(
+    event: Circus.Event,
+    state: Circus.State,
+  ): void | Promise<void>;
 }
 
 export type Module = NodeModule;

packages/jest-types/src/Circus.ts

@@ -31,12 +31,14 @@ export type Hook = {
   timeout: number | undefined | null;
 };
 
-export type EventHandler = (event: Event, state: State) => void;
+export interface EventHandler {
+  (event: AsyncEvent, state: State): void | Promise<void>;
+  (event: SyncEvent, state: State): void;
+}
 
-export type Event =
-  | {
-      name: 'include_test_location_in_result';
-    }
+export type Event = SyncEvent | AsyncEvent;
+
+export type SyncEvent =
   | {
       asyncError: Error;
       mode: BlockMode;
@@ -63,6 +65,23 @@ export type Event =
       mode?: TestMode;
       timeout: number | undefined;
     }
+  | {
+      // Any unhandled error that happened outside of test/hooks (unless it is
+      // an `afterAll` hook)
+      name: 'error';
+      error: Exception;
+    };
+
+export type AsyncEvent =
+  | {
+      // first action to dispatch. Good time to initialize all settings
+      name: 'setup';
+      testNamePattern?: string;
+      parentProcess: Process;
+    }
+  | {
+      name: 'include_test_location_in_result';
+    }
   | {
       name: 'hook_start';
       hook: Hook;
@@ -133,18 +152,6 @@ export type Event =
   | {
       name: 'run_finish';
     }
-  | {
-      // Any unhandled error that happened outside of test/hooks (unless it is
-      // an `afterAll` hook)
-      name: 'error';
-      error: Exception;
-    }
-  | {
-      // first action to dispatch. Good time to initialize all settings
-      name: 'setup';
-      testNamePattern?: string;
-      parentProcess: Process;
-    }
   | {
       // Action dispatched after everything is finished and we're about to wrap
       // things up and return test results to the parent process (caller).