chore: split run tests into run and debug (microsoft#37246)

Author: pavelfeldman

packages/playwright/src/index.ts

@@ -240,7 +240,7 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
       (testInfo as TestInfoImpl)._setDebugMode();
 
     playwright._defaultContextOptions = _combinedContextOptions;
-    playwright._defaultContextTimeout = process.env.PLAYWRIGHT_DEBUGGER_MCP ? 5000 : actionTimeout || 0;
+    playwright._defaultContextTimeout = process.env.PLAYWRIGHT_DEBUGGER_ENABLED ? 5000 : actionTimeout || 0;
     playwright._defaultContextNavigationTimeout = navigationTimeout || 0;
     await use();
     playwright._defaultContextOptions = undefined;

packages/playwright/src/mcp/browser/backend.ts

@@ -73,7 +73,7 @@ const doneToolSchema = defineToolSchema({
 });
 
 export async function runBrowserBackendOnError(page: playwright.Page, message: () => string) {
-  if (!process.env.PLAYWRIGHT_DEBUGGER_MCP)
+  if (!process.env.PLAYWRIGHT_DEBUGGER_ENABLED)
     return;
   const snapshot = await (page as PageEx)._snapshotForAI();
   const introMessage = `### Paused on error:
@@ -84,5 +84,5 @@ ${snapshot}
 
 ### Task
 Try recovering from the error prior to continuing, use following tools to recover: ${tools.map(tool => tool.schema.name).join(', ')}`;
-  await runOnPauseBackendLoop(process.env.PLAYWRIGHT_DEBUGGER_MCP!, new BrowserBackend(page), introMessage);
+  await runOnPauseBackendLoop(process.env.PLAYWRIGHT_MDB_URL!, new BrowserBackend(page), introMessage);
 }

packages/playwright/src/mcp/sdk/mdb.ts

@@ -153,7 +153,7 @@ export async function runMainBackend(backendFactory: mcpServer.ServerBackendFact
     create: () => mdbBackend
   };
   const url = await startAsHttp(factory, { port: options?.port || 0 });
-  process.env.PLAYWRIGHT_DEBUGGER_MCP = url;
+  process.env.PLAYWRIGHT_MDB_URL = url;
 
   if (options?.port !== undefined)
     return url;

packages/playwright/src/mcp/test/backend.ts

@@ -16,8 +16,7 @@
 
 import * as mcp from '../sdk/exports.js';
 import { Context } from './context';
-import { listTests } from './listTests';
-import { runTests } from './runTests';
+import { listTests, runTests, debugTest } from './tools.js';
 import { snapshot, pickLocator, evaluate } from '../browser/tools';
 
 import type { ConfigLocation } from '../../common/config';
@@ -27,7 +26,7 @@ import type { Tool } from './tool';
 export class TestServerBackend implements mcp.ServerBackend {
   readonly name = 'Playwright';
   readonly version = '0.0.1';
-  private _tools: Tool<any>[] = [listTests, runTests];
+  private _tools: Tool<any>[] = [listTests, runTests, debugTest];
   private _context: Context;
 
   constructor(resolvedLocation: ConfigLocation, options?: { muteConsole?: boolean }) {

packages/playwright/src/mcp/test/listTests.ts

@@ -0,0 +1,137 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { noColors } from 'playwright-core/lib/utils';
+
+import { z } from '../sdk/bundle';
+import { terminalScreen } from '../../reporters/base';
+import ListReporter from '../../reporters/list';
+import ListModeReporter from '../../reporters/listModeReporter';
+
+import { defineTool } from './tool';
+import { StringWriteStream } from './streams';
+
+export const listTests = defineTool({
+  schema: {
+    name: 'playwright_test_list_tests',
+    title: 'List tests',
+    description: 'List tests',
+    inputSchema: z.object({}),
+    type: 'readOnly',
+  },
+
+  handle: async context => {
+    const { screen, stream } = createScreen();
+    const reporter = new ListModeReporter({ screen, includeTestId: true });
+    const testRunner = await context.createTestRunner();
+    await testRunner.listTests(reporter, {});
+
+    return {
+      content: [{ type: 'text', text: stream.content() }],
+    };
+  },
+});
+
+export const runTests = defineTool({
+  schema: {
+    name: 'playwright_test_run_tests',
+    title: 'Run tests',
+    description: 'Run tests',
+    inputSchema: z.object({
+      locations: z.array(z.string()).describe('Folder, file or location to run: "test/e2e" or "test/e2e/file.spec.ts" or "test/e2e/file.spec.ts:20"'),
+      projects: z.array(z.string()).optional().describe('Projects to run, projects from playwright.config.ts, by default runs all projects. Running with "chromium" is a good start'),
+    }),
+    type: 'readOnly',
+  },
+
+  handle: async (context, params) => {
+    const { screen, stream } = createScreen();
+    const configDir = context.configLocation.configDir;
+    const reporter = new ListReporter({ configDir, screen, includeTestId: true });
+    const testRunner = await context.createTestRunner();
+    const result = await testRunner.runTests(reporter, {
+      locations: params.locations,
+      projects: params.projects,
+    });
+
+    const text = stream.content();
+    return {
+      content: [
+        { type: 'text', text },
+      ],
+      isError: result.status !== 'passed',
+    };
+  },
+});
+
+export const debugTest = defineTool({
+  schema: {
+    name: 'playwright_test_debug_test',
+    title: 'Debug single test',
+    description: 'Debug single test',
+    inputSchema: z.object({
+      test: z.object({
+        id: z.string().describe('Test ID to debug.'),
+        title: z.string().describe('Human readable test title for granting permission to debug the test.'),
+      }),
+    }),
+    type: 'readOnly',
+  },
+
+  handle: async (context, params) => {
+    const stream = new StringWriteStream();
+    const screen = {
+      ...terminalScreen,
+      isTTY: false,
+      colors: noColors,
+      stdout: stream as unknown as NodeJS.WriteStream,
+      stderr: stream as unknown as NodeJS.WriteStream,
+    };
+    const configDir = context.configLocation.configDir;
+    const reporter = new ListReporter({ configDir, screen });
+    const testRunner = await context.createTestRunner();
+    process.env.PLAYWRIGHT_DEBUGGER_ENABLED = '1';
+    const result = await testRunner.runTests(reporter, {
+      headed: true,
+      testIds: [params.test.id],
+      // For automatic recovery
+      timeout: 0,
+      workers: 1,
+    }).finally(() => {
+      process.env.PLAYWRIGHT_DEBUGGER_ENABLED = undefined;
+    });
+
+    const text = stream.content();
+    return {
+      content: [
+        { type: 'text', text },
+      ],
+      isError: result.status !== 'passed',
+    };
+  },
+});
+
+function createScreen() {
+  const stream = new StringWriteStream();
+  const screen = {
+    ...terminalScreen,
+    isTTY: false,
+    colors: noColors,
+    stdout: stream as unknown as NodeJS.WriteStream,
+    stderr: stream as unknown as NodeJS.WriteStream,
+  };
+  return { screen, stream };
+}