chore: add mcp tools test harness (#35260)

package-lock.json

@@ -10074,12 +10074,23 @@
         "mcp": "cli.js"
       },
       "devDependencies": {
-        "@modelcontextprotocol/sdk": "^1.6.1"
+        "@modelcontextprotocol/sdk": "^1.6.1",
+        "@types/node": "^22.13.10"
       },
       "engines": {
         "node": ">=18"
       }
     },
+    "packages/playwright-mcp/node_modules/@types/node": {
+      "version": "22.13.10",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.10.tgz",
+      "integrity": "sha512-I6LPUvlRH+O6VRUqYOcMudhaIdUVWfsjnZavnsraHvpBwaEyMN29ry+0UVJhImYL16xsscu0aske3yA+uPOWfw==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "undici-types": "~6.20.0"
+      }
+    },
     "packages/playwright-mcp/node_modules/commander": {
       "version": "13.1.0",
       "resolved": "https://registry.npmjs.org/commander/-/commander-13.1.0.tgz",
@@ -10089,6 +10100,13 @@
         "node": ">=18"
       }
     },
+    "packages/playwright-mcp/node_modules/undici-types": {
+      "version": "6.20.0",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz",
+      "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==",
+      "dev": true,
+      "license": "MIT"
+    },
     "packages/playwright-mcp/node_modules/zod": {
       "version": "3.24.2",
       "resolved": "https://registry.npmjs.org/zod/-/zod-3.24.2.tgz",

package.json

@@ -28,6 +28,7 @@
     "test-html-reporter": "playwright test --config=packages/html-reporter",
     "test-web": "playwright test --config=packages/web",
     "ttest": "node ./tests/playwright-test/stable-test-runner/node_modules/@playwright/test/cli test --config=tests/playwright-test/playwright.config.ts",
+    "mtest": "playwright test --config=packages/playwright-mcp",
     "ct": "playwright test tests/components/test-all.spec.js --reporter=list",
     "test": "playwright test --config=tests/library/playwright.config.ts",
     "eslint": "eslint --cache",

packages/playwright-mcp/README.md

@@ -109,27 +109,27 @@ The Playwright MCP provides a set of tools for browser automation. Here are all
 - **browser_click**
   - Description: Perform click on a web page
   - Parameters:
-    - `element` (string): Human-readable element description used to obtain the permission to interact with the element
+    - `element` (string): Human-readable element description used to obtain permission to interact with the element
     - `ref` (string): Exact target element reference from the page snapshot
 
 - **browser_hover**
   - Description: Hover over element on page
   - Parameters:
-    - `element` (string): Human-readable element description used to obtain the permission to interact with the element
+    - `element` (string): Human-readable element description used to obtain permission to interact with the element
     - `ref` (string): Exact target element reference from the page snapshot
 
 - **browser_drag**
   - Description: Perform drag and drop between two elements
   - Parameters:
-    - `startElement` (string): Human-readable source element description used to obtain the permission to interact with the element
+    - `startElement` (string): Human-readable source element description used to obtain permission to interact with the element
     - `startRef` (string): Exact source element reference from the page snapshot
-    - `endElement` (string): Human-readable target element description used to obtain the permission to interact with the element
+    - `endElement` (string): Human-readable target element description used to obtain permission to interact with the element
     - `endRef` (string): Exact target element reference from the page snapshot
 
 - **browser_type**
   - Description: Type text into editable element
   - Parameters:
-    - `element` (string): Human-readable element description used to obtain the permission to interact with the element
+    - `element` (string): Human-readable element description used to obtain permission to interact with the element
     - `ref` (string): Exact target element reference from the page snapshot
     - `text` (string): Text to type into the element
     - `submit` (boolean): Whether to submit entered text (press Enter after)

packages/playwright-mcp/package.json

@@ -29,7 +29,8 @@
     "zod-to-json-schema": "^3.24.4"
   },
   "devDependencies": {
-    "@modelcontextprotocol/sdk": "^1.6.1"
+    "@modelcontextprotocol/sdk": "^1.6.1",
+    "@types/node": "^22.13.10"
   },
   "bin": {
     "mcp": "cli.js"

packages/playwright-mcp/playwright.config.ts

@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  testDir: './tests',
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 2 : 0,
+  workers: process.env.CI ? 1 : undefined,
+  reporter: 'list',
+  projects: [{ name: 'default' }],
+});

packages/playwright-mcp/src/tools/screenshot.ts

@@ -38,7 +38,7 @@ export const screenshot: Tool = {
 };
 
 const elementSchema = z.object({
-  element: z.string().describe('Human-readable element description used to obtain the permission to interact with the element'),
+  element: z.string().describe('Human-readable element description used to obtain permission to interact with the element'),
 });
 
 const moveMouseSchema = elementSchema.extend({

packages/playwright-mcp/src/tools/snapshot.ts

@@ -35,7 +35,7 @@ export const snapshot: Tool = {
 };
 
 const elementSchema = z.object({
-  element: z.string().describe('Human-readable element description used to obtain the permission to interact with the element'),
+  element: z.string().describe('Human-readable element description used to obtain permission to interact with the element'),
   ref: z.string().describe('Exact target element reference from the page snapshot'),
 });
 

packages/playwright-mcp/src/tools/utils.ts

@@ -81,10 +81,10 @@ export async function captureAriaSnapshot(page: playwright.Page): Promise<ToolRe
   const snapshot = await page.locator('html').ariaSnapshot({ ref: true });
   return {
     content: [{ type: 'text', text: `
-      # Page URL: ${page.url()}
-      # Page Title: ${page.title()}
-      # Page Snapshot
-      ${snapshot}`
+# Page URL: ${page.url()}
+# Page Title: ${page.title()}
+# Page Snapshot
+${snapshot}`
     }],
   };
 }

packages/playwright-mcp/tests/basic.spec.ts

@@ -0,0 +1,163 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import path from 'path';
+
+import { test, expect } from '@playwright/test';
+
+import { MCPServer } from './fixtures';
+
+async function startServer(): Promise<MCPServer> {
+  const server = new MCPServer('node', [path.join(__dirname, '../cli.js'), '--headless']);
+  const initialize = await server.send({
+    jsonrpc: '2.0',
+    id: 0,
+    method: 'initialize',
+    params: {
+      protocolVersion: '2024-11-05',
+      capabilities: {},
+      clientInfo: {
+        name: 'Playwright Test',
+        version: '0.0.0',
+      },
+    },
+  });
+
+  expect(initialize).toEqual(expect.objectContaining({
+    id: 0,
+    result: expect.objectContaining({
+      protocolVersion: '2024-11-05',
+      capabilities: {
+        tools: {},
+        resources: {},
+      },
+      serverInfo: expect.objectContaining({
+        name: 'Playwright',
+        version: expect.any(String),
+      }),
+    }),
+  }));
+
+  await server.sendNoReply({
+    jsonrpc: '2.0',
+    method: 'notifications/initialized',
+  });
+
+  return server;
+}
+
+test('test tool list', async ({}) => {
+  const server = await startServer();
+
+  const list = await server.send({
+    jsonrpc: '2.0',
+    id: 1,
+    method: 'tools/list',
+  });
+
+  expect(list).toEqual(expect.objectContaining({
+    id: 1,
+    result: expect.objectContaining({
+      tools: [
+        expect.objectContaining({
+          name: 'browser_navigate',
+        }),
+        expect.objectContaining({
+          name: 'browser_go_back',
+        }),
+        expect.objectContaining({
+          name: 'browser_go_forward',
+        }),
+        expect.objectContaining({
+          name: 'browser_snapshot',
+        }),
+        expect.objectContaining({
+          name: 'browser_click',
+        }),
+        expect.objectContaining({
+          name: 'browser_hover',
+        }),
+        expect.objectContaining({
+          name: 'browser_type',
+        }),
+        expect.objectContaining({
+          name: 'browser_press_key',
+        }),
+        expect.objectContaining({
+          name: 'browser_wait',
+        }),
+        expect.objectContaining({
+          name: 'browser_save_as_pdf',
+        }),
+        expect.objectContaining({
+          name: 'browser_close',
+        }),
+      ],
+    }),
+  }));
+});
+
+test('test resources list', async ({}) => {
+  const server = await startServer();
+
+  const list = await server.send({
+    jsonrpc: '2.0',
+    id: 2,
+    method: 'resources/list',
+  });
+
+  expect(list).toEqual(expect.objectContaining({
+    id: 2,
+    result: expect.objectContaining({
+      resources: [
+        expect.objectContaining({
+          uri: 'browser://console',
+          mimeType: 'text/plain',
+        }),
+      ],
+    }),
+  }));
+});
+
+test('test browser_navigate', async ({}) => {
+  const server = await startServer();
+
+  const response = await server.send({
+    jsonrpc: '2.0',
+    id: 2,
+    method: 'tools/call',
+    params: {
+      name: 'browser_navigate',
+      arguments: {
+        url: 'https://example.com',
+      },
+    },
+  });
+
+  expect(response).toEqual(expect.objectContaining({
+    id: 2,
+    result: {
+      content: [{
+        type: 'text',
+        text: expect.stringContaining(`
+# Page URL: https://example.com/
+# Page Title: [object Promise]
+# Page Snapshot
+- document`),
+      }],
+    },
+  }));
+});

packages/playwright-mcp/tests/fixtures.ts

@@ -0,0 +1,104 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { spawn } from 'child_process';
+import EventEmitter from 'events';
+
+import type { ChildProcess } from 'child_process';
+
+export class MCPServer extends EventEmitter {
+  private _child: ChildProcess;
+  private _messageQueue: any[] = [];
+  private _messageResolvers: ((value: any) => void)[] = [];
+  private _buffer: string = '';
+
+  constructor(command: string, args: string[]) {
+    super();
+    this._child = spawn(command, args, {
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+
+    this._child.stdout?.on('data', data => {
+      this._buffer += data.toString();
+      let newlineIndex: number;
+
+      while ((newlineIndex = this._buffer.indexOf('\n')) !== -1) {
+        const message = this._buffer.slice(0, newlineIndex).trim();
+        this._buffer = this._buffer.slice(newlineIndex + 1);
+
+        if (!message)
+          continue;
+
+        const parsed = JSON.parse(message);
+        if (this._messageResolvers.length > 0) {
+          const resolve = this._messageResolvers.shift();
+          resolve!(parsed);
+        } else {
+          this._messageQueue.push(parsed);
+        }
+      }
+    });
+
+    this._child.stderr?.on('data', data => {
+      throw new Error('Server stderr:', data.toString());
+    });
+
+    this._child.on('exit', code => {
+      if (code !== 0)
+        throw new Error(`Server exited with code ${code}`);
+    });
+  }
+
+  async send(message: any, options?: { timeout?: number }): Promise<void> {
+    await this.sendNoReply(message);
+    return this._waitForResponse(options || {});
+  }
+
+  async sendNoReply(message: any): Promise<void> {
+    const jsonMessage = JSON.stringify(message) + '\n';
+    await new Promise<void>((resolve, reject) => {
+      this._child.stdin?.write(jsonMessage, err => {
+        if (err)
+          reject(err);
+        else
+          resolve();
+      });
+    });
+  }
+
+  private async _waitForResponse(options: { timeout?: number }): Promise<any> {
+    if (this._messageQueue.length > 0)
+      return this._messageQueue.shift();
+
+    return new Promise((resolve, reject) => {
+      const timeoutId = setTimeout(() => {
+        reject(new Error('Timeout waiting for message'));
+      }, options.timeout || 5000);
+
+      this._messageResolvers.push(message => {
+        clearTimeout(timeoutId);
+        resolve(message);
+      });
+    });
+  }
+
+  async close(): Promise<void> {
+    return new Promise(resolve => {
+      this._child.on('exit', () => resolve());
+      this._child.stdin?.end();
+    });
+  }
+}