feat(web-integration): add browser page selection actions

Author: quanru

apps/site/docs/en/web-api-reference.mdx

@@ -83,6 +83,7 @@ const agent = new PuppeteerBrowserAgent(browser, page, {
 - `newPage()` — Create a new page and make it active.
 - `setActivePage(page: Page)` — Explicitly set which Puppeteer page the Browser Agent controls next.
 - `waitForNewPage(action?, options?)` — Wait for a newly opened page without implicitly switching the active page.
+- AI page selection actions — Browser Agent exposes `ListBrowserPages` and `SetActivePage` in its action space, so AI plans can inspect open tabs and switch the active page by index, title, or URL. These actions are not available on Page Agent.
 
 ### Examples
 
@@ -193,6 +194,7 @@ const agent = new PlaywrightBrowserAgent(context, page, {
 - `newPage()` — Create a new page and make it active.
 - `setActivePage(page: Page)` — Explicitly set which Playwright page the Browser Agent controls next.
 - `waitForNewPage(action?, options?)` — Wait for a newly opened page without implicitly switching the active page.
+- AI page selection actions — Browser Agent exposes `ListBrowserPages` and `SetActivePage` in its action space, so AI plans can inspect open tabs and switch the active page by index, title, or URL. These actions are not available on Page Agent.
 
 ### Examples
 

apps/site/docs/zh/web-api-reference.mdx

@@ -83,6 +83,7 @@ const agent = new PuppeteerBrowserAgent(browser, page, {
 - `newPage()` —— 创建新页面并将其设为 active page。
 - `setActivePage(page: Page)` —— 显式指定 Browser Agent 接下来控制哪个 Puppeteer 页面。
 - `waitForNewPage(action?, options?)` —— 等待新打开的页面，但不会隐式切换 active page。
+- AI 页面选择 action —— Browser Agent 会在 actionSpace 中暴露 `ListBrowserPages` 和 `SetActivePage`，AI 计划可以先查看已打开的标签页，再按 index、title 或 URL 切换 active page。这些 action 不会出现在 Page Agent 上。
 
 ### 示例
 
@@ -193,6 +194,7 @@ const agent = new PlaywrightBrowserAgent(context, page, {
 - `newPage()` —— 创建新页面并将其设为 active page。
 - `setActivePage(page: Page)` —— 显式指定 Browser Agent 接下来控制哪个 Playwright 页面。
 - `waitForNewPage(action?, options?)` —— 等待新打开的页面，但不会隐式切换 active page。
+- AI 页面选择 action —— Browser Agent 会在 actionSpace 中暴露 `ListBrowserPages` 和 `SetActivePage`，AI 计划可以先查看已打开的标签页，再按 index、title 或 URL 切换 active page。这些 action 不会出现在 Page Agent 上。
 
 ### 示例
 

packages/web-integration/package.json

@@ -123,7 +123,8 @@
     "semver": "7.5.2",
     "socket.io": "^4.8.1",
     "socket.io-client": "4.8.1",
-    "ws": "^8.18.1"
+    "ws": "^8.18.1",
+    "zod": "^3.25.1"
   },
   "devDependencies": {
     "@playwright/test": "^1.45.0",

packages/web-integration/src/common/browser-agent.ts

@@ -1,10 +1,14 @@
+import type { DeviceAction } from '@midscene/core';
 import type { DebugFunction } from '@midscene/shared/logger';
+import { z } from 'zod';
 
 export type BrowserAgentAdapter<Page, NewPageEvent> = {
   pages(): Page[] | Promise<Page[]>;
   newPage(): Promise<Page>;
   isPageClosed(page: Page): boolean;
   bringToFront(page: Page): Promise<void> | void;
+  pageTitle(page: Page): Promise<string> | string;
+  pageUrl(page: Page): string;
   onNewPage(handler: (event: NewPageEvent) => void): void;
   offNewPage(handler: (event: NewPageEvent) => void): void;
   resolveNewPage(event: NewPageEvent): Page | Promise<Page | null> | null;
@@ -21,6 +25,51 @@ export type BrowserAgentPageControllerOptions<Page, NewPageEvent> = {
   debug: DebugFunction;
 };
 
+export type BrowserAgentPageSummary = {
+  index: number;
+  active: boolean;
+  title: string;
+  url: string;
+};
+
+const setActivePageParamSchema = z.object({
+  index: z
+    .number()
+    .int()
+    .min(0)
+    .optional()
+    .describe('0-based page/tab index returned by ListBrowserPages.'),
+  title: z
+    .string()
+    .optional()
+    .describe('Case-insensitive page title substring to match.'),
+  url: z
+    .string()
+    .optional()
+    .describe('Case-insensitive page URL substring to match.'),
+});
+
+export type SetActivePageParam = z.infer<typeof setActivePageParamSchema>;
+
+const normalizeOptionalText = (value: string | undefined) => {
+  const trimmed = value?.trim();
+  return trimmed ? trimmed.toLowerCase() : undefined;
+};
+
+const describeSelector = (selector: SetActivePageParam) => {
+  const parts: string[] = [];
+  if (selector.index !== undefined) {
+    parts.push(`index ${selector.index}`);
+  }
+  if (selector.title?.trim()) {
+    parts.push(`title "${selector.title.trim()}"`);
+  }
+  if (selector.url?.trim()) {
+    parts.push(`url "${selector.url.trim()}"`);
+  }
+  return parts.join(', ');
+};
+
 export class BrowserAgentPageController<Page, NewPageEvent> {
   private readonly agentName: string;
   private readonly adapter: BrowserAgentAdapter<Page, NewPageEvent>;
@@ -54,6 +103,17 @@ export class BrowserAgentPageController<Page, NewPageEvent> {
     return this.adapter.pages();
   }
 
+  async pageSummaries(): Promise<BrowserAgentPageSummary[]> {
+    const pages = await this.adapter.pages();
+    const activePage = this.activePage;
+
+    return Promise.all(
+      pages.map((page, index) =>
+        this.pageSummary(page, index, page === activePage),
+      ),
+    );
+  }
+
   async newPage() {
     const page = await this.adapter.newPage();
     await this.setActivePage(page);
@@ -75,6 +135,68 @@ export class BrowserAgentPageController<Page, NewPageEvent> {
     }
   }
 
+  async setActivePageBySelector(
+    selector: SetActivePageParam,
+  ): Promise<BrowserAgentPageSummary> {
+    const hasIndex = selector.index !== undefined;
+    const title = normalizeOptionalText(selector.title);
+    const url = normalizeOptionalText(selector.url);
+
+    if (!hasIndex && !title && !url) {
+      throw new Error(
+        `[midscene] SetActivePage requires index, title, or url for ${this.agentName}.`,
+      );
+    }
+
+    const pages = await this.adapter.pages();
+
+    if (hasIndex) {
+      const page = pages[selector.index as number];
+      if (!page || this.adapter.isPageClosed(page)) {
+        throw new Error(
+          `[midscene] Cannot find ${this.agentName} page with index ${selector.index}. Available page indexes: ${pages
+            .map((_, index) => index)
+            .join(', ')}`,
+        );
+      }
+
+      await this.setActivePage(page);
+      return this.pageSummary(page, selector.index as number, true);
+    }
+
+    const matchedPages: Array<{ page: Page; index: number }> = [];
+    for (let index = 0; index < pages.length; index++) {
+      const page = pages[index];
+      if (this.adapter.isPageClosed(page)) {
+        continue;
+      }
+
+      const summary = await this.pageSummary(page, index, false);
+      const matchedTitle =
+        !title || summary.title.toLowerCase().includes(title);
+      const matchedUrl = !url || summary.url.toLowerCase().includes(url);
+      if (matchedTitle && matchedUrl) {
+        matchedPages.push({ page, index });
+      }
+    }
+
+    if (matchedPages.length === 0) {
+      throw new Error(
+        `[midscene] Cannot find ${this.agentName} page matching ${describeSelector(selector)}.`,
+      );
+    }
+
+    if (matchedPages.length > 1) {
+      throw new Error(
+        `[midscene] Multiple ${this.agentName} pages matched ${describeSelector(selector)}. Use ListBrowserPages and pass an index to SetActivePage.`,
+      );
+    }
+
+    const { page, index } = matchedPages[0];
+    await this.setActivePage(page);
+    return this.pageSummary(page, index, true);
+  }
+
   async waitForNewPage(
     action?: () => Promise<unknown> | unknown,
     opts?: { timeout?: number },
@@ -94,6 +216,34 @@ export class BrowserAgentPageController<Page, NewPageEvent> {
     this.adapter.offNewPage(this.newPageHandler);
   }
 
+  private async pageSummary(
+    page: Page,
+    index: number,
+    active: boolean,
+  ): Promise<BrowserAgentPageSummary> {
+    let title = '';
+    let url = '';
+
+    try {
+      title = await this.adapter.pageTitle(page);
+    } catch (error) {
+      this.debug(`failed to read page title: ${error}`);
+    }
+
+    try {
+      url = this.adapter.pageUrl(page);
+    } catch (error) {
+      this.debug(`failed to read page url: ${error}`);
+    }
+
+    return {
+      index,
+      active,
+      title,
+      url,
+    };
+  }
+
   private async followNewPage(event: NewPageEvent) {
     if (!this.isNewPageEvent(event)) {
       return;
@@ -165,3 +315,41 @@ export class BrowserAgentPageController<Page, NewPageEvent> {
     return { promise, dispose };
   }
 }
+
+export const createBrowserAgentPageActions = <Page, NewPageEvent>(options: {
+  agentName: string;
+  getPageController: () => BrowserAgentPageController<Page, NewPageEvent>;
+}): DeviceAction<any>[] => [
+  {
+    name: 'ListBrowserPages',
+    description:
+      'List all open browser pages/tabs and show which one is currently active. Use this before switching pages when a task refers to another tab or window.',
+    call: async () => options.getPageController().pageSummaries(),
+  },
+  {
+    name: 'SetActivePage',
+    description:
+      'Set the active browser page/tab by 0-based index, title substring, or URL substring. Use index from ListBrowserPages when more than one page could match.',
+    paramSchema: setActivePageParamSchema,
+    sample: {
+      index: 1,
+    },
+    call: async (param) =>
+      options.getPageController().setActivePageBySelector(param),
+  },
+];
+
+export const appendBrowserAgentPageActions = (
+  customActions: DeviceAction<any>[] | undefined,
+  browserActions: DeviceAction<any>[],
+) => {
+  if (!customActions?.length) {
+    return browserActions;
+  }
+
+  const customActionNames = new Set(customActions.map((action) => action.name));
+  return [
+    ...customActions,
+    ...browserActions.filter((action) => !customActionNames.has(action.name)),
+  ];
+};

packages/web-integration/src/playwright/browser-agent.ts

@@ -1,6 +1,8 @@
 import {
   type BrowserAgentAdapter,
   BrowserAgentPageController,
+  appendBrowserAgentPageActions,
+  createBrowserAgentPageActions,
 } from '@/common/browser-agent';
 import {
   applyForceChromeSelectRendering,
@@ -24,6 +26,8 @@ const createPlaywrightBrowserAdapter = (
   newPage: () => context.newPage(),
   isPageClosed: (page) => page.isClosed(),
   bringToFront: (page) => page.bringToFront(),
+  pageTitle: (page) => page.title(),
+  pageUrl: (page) => page.url(),
   onNewPage: (handler) => context.on('page', handler),
   offNewPage: (handler) => context.off('page', handler),
   resolveNewPage: (page) => page,
@@ -73,13 +77,32 @@ export class PlaywrightBrowserAgent extends PageAgent<PlaywrightWebPage> {
       ...agentOpts
     } = opts ?? {};
     const { forceChromeSelectRendering } = agentOpts;
+    const pageControllerRef: {
+      current?: BrowserAgentPageController<PlaywrightPage, PlaywrightPage>;
+    } = {};
+    const getPageController = () => {
+      if (!pageControllerRef.current) {
+        throw new Error(
+          '[midscene] PlaywrightBrowserAgent page controller is not initialized.',
+        );
+      }
+      return pageControllerRef.current;
+    };
+    const browserActions = createBrowserAgentPageActions({
+      agentName: 'PlaywrightBrowserAgent',
+      getPageController,
+    });
     const webPage = new PlaywrightWebPage(initialPage, {
       ...agentOpts,
       forceSameTabNavigation: false,
+      customActions: appendBrowserAgentPageActions(
+        agentOpts.customActions,
+        browserActions,
+      ),
     });
     super(webPage, agentOpts);
 
-    this.pageController = new BrowserAgentPageController({
+    const pageController = new BrowserAgentPageController({
       agentName: 'PlaywrightBrowserAgent',
       adapter: createPlaywrightBrowserAdapter(context),
       getActivePage: () => this.interface.underlyingPage as PlaywrightPage,
@@ -90,6 +113,8 @@ export class PlaywrightBrowserAgent extends PageAgent<PlaywrightWebPage> {
       newPageTimeout,
       debug,
     });
+    pageControllerRef.current = pageController;
+    this.pageController = pageController;
 
     applyForceChromeSelectRendering(
       initialPage,

packages/web-integration/src/puppeteer/browser-agent.ts

@@ -1,6 +1,8 @@
 import {
   type BrowserAgentAdapter,
   BrowserAgentPageController,
+  appendBrowserAgentPageActions,
+  createBrowserAgentPageActions,
 } from '@/common/browser-agent';
 import {
   applyForceChromeSelectRendering,
@@ -25,6 +27,8 @@ const createPuppeteerBrowserAdapter = (
   newPage: () => browser.newPage(),
   isPageClosed: (page) => page.isClosed(),
   bringToFront: (page) => page.bringToFront(),
+  pageTitle: (page) => page.title(),
+  pageUrl: (page) => page.url(),
   onNewPage: (handler) => browser.on('targetcreated', handler),
   offNewPage: (handler) => browser.off('targetcreated', handler),
   isNewPageEvent: (target) => target.type() === 'page',
@@ -75,13 +79,32 @@ export class PuppeteerBrowserAgent extends PageAgent<PuppeteerWebPage> {
       ...agentOpts
     } = opts ?? {};
     const { forceChromeSelectRendering } = agentOpts;
+    const pageControllerRef: {
+      current?: BrowserAgentPageController<PuppeteerPage, PuppeteerTarget>;
+    } = {};
+    const getPageController = () => {
+      if (!pageControllerRef.current) {
+        throw new Error(
+          '[midscene] PuppeteerBrowserAgent page controller is not initialized.',
+        );
+      }
+      return pageControllerRef.current;
+    };
+    const browserActions = createBrowserAgentPageActions({
+      agentName: 'PuppeteerBrowserAgent',
+      getPageController,
+    });
     const webPage = new PuppeteerWebPage(initialPage, {
       ...agentOpts,
       forceSameTabNavigation: false,
+      customActions: appendBrowserAgentPageActions(
+        agentOpts.customActions,
+        browserActions,
+      ),
     });
     super(webPage, agentOpts);
 
-    this.pageController = new BrowserAgentPageController({
+    const pageController = new BrowserAgentPageController({
       agentName: 'PuppeteerBrowserAgent',
       adapter: createPuppeteerBrowserAdapter(browser),
       getActivePage: () => this.interface.underlyingPage as PuppeteerPage,
@@ -92,6 +115,8 @@ export class PuppeteerBrowserAgent extends PageAgent<PuppeteerWebPage> {
       newPageTimeout,
       debug,
     });
+    pageControllerRef.current = pageController;
+    this.pageController = pageController;
 
     applyForceChromeSelectRendering(
       initialPage,