From 6430818e027a282468ef1f0b15b56ae79609c94d Mon Sep 17 00:00:00 2001 From: zzcn Date: Sun, 15 Mar 2026 10:51:22 +0800 Subject: [PATCH 1/7] fix(web-integration): auto switch to new tab when forceSameTabNavigation is false When a click opens a new browser tab, the agent now automatically switches its underlying page reference to the new tab so that subsequent AI actions operate on it. This only applies when forceSameTabNavigation is explicitly set to false. Closes #2098 --- .../web-integration/src/playwright/index.ts | 3 ++ .../src/puppeteer/base-page.ts | 49 +++++++++++++++++++ .../web-integration/src/puppeteer/index.ts | 3 ++ .../ai/web/puppeteer/tab-navigation.test.ts | 10 ++-- 4 files changed, 59 insertions(+), 6 deletions(-) diff --git a/packages/web-integration/src/playwright/index.ts b/packages/web-integration/src/playwright/index.ts index 92d5f6496e..02c51d6172 100644 --- a/packages/web-integration/src/playwright/index.ts +++ b/packages/web-integration/src/playwright/index.ts @@ -15,6 +15,7 @@ import { getDebug } from '@midscene/shared/logger'; import semver from 'semver'; import { BROWSER_NAVIGATION_ERROR_PATTERN, + autoSwitchToNewTab, forceChromeSelectRendering as applyChromeSelectRendering, forceClosePopup, } from '../puppeteer/base-page'; @@ -57,6 +58,8 @@ export class PlaywrightAgent extends PageAgent { if (forceSameTabNavigation) { forceClosePopup(page, debug); + } else { + autoSwitchToNewTab(page, webPage, debug); } if (forceChromeSelectRendering) { diff --git a/packages/web-integration/src/puppeteer/base-page.ts b/packages/web-integration/src/puppeteer/base-page.ts index f12b4c6ffc..866a0b6a4c 100644 --- a/packages/web-integration/src/puppeteer/base-page.ts +++ b/packages/web-integration/src/puppeteer/base-page.ts @@ -1265,6 +1265,55 @@ export function forceClosePopup( }); } +/** + * When forceSameTabNavigation is false, automatically switch the agent's + * underlying page reference to newly opened tabs so that subsequent actions + * operate on the new tab instead of the original one. + */ +export function autoSwitchToNewTab( + page: PuppeteerPage | PlaywrightPage, + webPage: Page, + debugProfile: DebugFunction, +) { + page.on('popup', async (popup) => { + if (!popup) { + console.warn( + '[midscene] got a popup event, but the popup is not ready yet, skip', + ); + return; + } + + try { + const popupPage = popup as PuppeteerPage; + if (popupPage.isClosed()) { + debugProfile('popup is already closed, skip switching'); + return; + } + + const url = popupPage.url(); + debugProfile(`New tab detected: ${url}, switching to it`); + + // Update the underlying page reference to the new tab + webPage.underlyingPage = popup as any; + + // Bring the new tab to the front + await popupPage.bringToFront(); + + // Recursively listen for popups on the new tab as well + autoSwitchToNewTab( + popup as PuppeteerPage | PlaywrightPage, + webPage, + debugProfile, + ); + } catch (error) { + debugProfile(`failed to switch to new tab: ${error}`); + console.warn( + `[midscene:warning] Failed to switch to newly opened tab: ${error}`, + ); + } + }); +} + /** * Force Chrome to render select elements using base-select appearance instead of OS-native rendering. * This makes select elements visible in screenshots captured by Playwright/Puppeteer. diff --git a/packages/web-integration/src/puppeteer/index.ts b/packages/web-integration/src/puppeteer/index.ts index b9060478be..440846cf37 100644 --- a/packages/web-integration/src/puppeteer/index.ts +++ b/packages/web-integration/src/puppeteer/index.ts @@ -6,6 +6,7 @@ import semver from 'semver'; import { getWebpackRequire } from '../utils'; import { BROWSER_NAVIGATION_ERROR_PATTERN, + autoSwitchToNewTab, forceChromeSelectRendering as applyChromeSelectRendering, forceClosePopup, } from './base-page'; @@ -51,6 +52,8 @@ export class PuppeteerAgent extends PageAgent { if (forceSameTabNavigation) { forceClosePopup(page, debug); + } else { + autoSwitchToNewTab(page, webPage, debug); } if (forceChromeSelectRendering) { diff --git a/packages/web-integration/tests/ai/web/puppeteer/tab-navigation.test.ts b/packages/web-integration/tests/ai/web/puppeteer/tab-navigation.test.ts index a9e1221aaf..ff3967f8bf 100644 --- a/packages/web-integration/tests/ai/web/puppeteer/tab-navigation.test.ts +++ b/packages/web-integration/tests/ai/web/puppeteer/tab-navigation.test.ts @@ -13,7 +13,7 @@ describe( () => { const ctx = createTestContext(); - it('not tracking active tab', async () => { + it('auto switch to new tab', async () => { const htmlPath = getFixturePath('tab-navigation.html'); const { originPage, reset } = await launchPage(`file://${htmlPath}`); ctx.resetFn = reset; @@ -25,11 +25,9 @@ describe( }); await sleep(3000); - // When forceSameTabNavigation is false, the agent should NOT follow the new tab - // So the weather forecast (which appears in the new tab) should NOT be visible - await expect(async () => { - await ctx.agent!.aiAssert('There is a weather forecast in the page'); - }).rejects.toThrowError(); + // When forceSameTabNavigation is false, the agent should automatically + // switch to the newly opened tab so subsequent actions operate on it + await ctx.agent.aiWaitFor('There is a weather forecast in the page'); }); it('tracking active tab', async () => { From 46a473779836435df89efe3ff67da7cc5b87b9f2 Mon Sep 17 00:00:00 2001 From: quanru Date: Mon, 8 Jun 2026 19:45:08 +0800 Subject: [PATCH 2/7] fix(web-integration): add browser agent page following --- .../docs/en/automate-with-scripts-in-yaml.mdx | 4 + .../docs/en/integrate-with-playwright.mdx | 10 +- .../site/docs/en/integrate-with-puppeteer.mdx | 10 +- apps/site/docs/en/web-api-reference.mdx | 40 +++- .../docs/zh/automate-with-scripts-in-yaml.mdx | 4 + .../docs/zh/integrate-with-playwright.mdx | 10 +- .../site/docs/zh/integrate-with-puppeteer.mdx | 10 +- apps/site/docs/zh/web-api-reference.mdx | 40 +++- packages/core/src/yaml.ts | 3 +- .../web-integration/src/common/web-agent.ts | 72 ++++++ packages/web-integration/src/index.ts | 14 +- .../web-integration/src/playwright/agent.ts | 41 ++++ .../src/playwright/ai-fixture.ts | 29 ++- .../src/playwright/browser-agent.ts | 193 ++++++++++++++++ .../web-integration/src/playwright/index.ts | 74 +----- .../src/puppeteer/agent-launcher.ts | 34 ++- .../web-integration/src/puppeteer/agent.ts | 41 ++++ .../src/puppeteer/base-page.ts | 49 ---- .../src/puppeteer/browser-agent.ts | 211 ++++++++++++++++++ .../web-integration/src/puppeteer/index.ts | 74 +----- .../ai/web/puppeteer/tab-navigation.test.ts | 116 ++++++++-- .../tests/ai/web/puppeteer/test-utils.ts | 4 +- 22 files changed, 851 insertions(+), 232 deletions(-) create mode 100644 packages/web-integration/src/common/web-agent.ts create mode 100644 packages/web-integration/src/playwright/agent.ts create mode 100644 packages/web-integration/src/playwright/browser-agent.ts create mode 100644 packages/web-integration/src/puppeteer/agent.ts create mode 100644 packages/web-integration/src/puppeteer/browser-agent.ts diff --git a/apps/site/docs/en/automate-with-scripts-in-yaml.mdx b/apps/site/docs/en/automate-with-scripts-in-yaml.mdx index 64502b15d2..2e362821bf 100644 --- a/apps/site/docs/en/automate-with-scripts-in-yaml.mdx +++ b/apps/site/docs/en/automate-with-scripts-in-yaml.mdx @@ -197,6 +197,10 @@ web: # Whether to restrict page navigation to the current tab, optional, defaults to true. forceSameTabNavigation: + # Whether to use BrowserAgent and automatically continue in newly opened pages, optional, defaults to false. + # Cannot be used together with forceSameTabNavigation: true. + autoFollowNewPage: + # CDP endpoint, optional. Connects to an existing browser instance via CDP instead of launching a new one. # Mutually exclusive with bridgeMode. cdpEndpoint: ws://localhost:9222/devtools/browser diff --git a/apps/site/docs/en/integrate-with-playwright.mdx b/apps/site/docs/en/integrate-with-playwright.mdx index b6822b98e9..e412d4f4b2 100644 --- a/apps/site/docs/en/integrate-with-playwright.mdx +++ b/apps/site/docs/en/integrate-with-playwright.mdx @@ -267,14 +267,18 @@ After the command executes successfully, it will output: `Midscene - report file Each Agent instance is bound to a single page. To make debugging easier, Midscene intercepts new tabs by default (for example, links with `target="_blank"`) and opens them in the current page. -If you want to restore opening in a new tab, set `forceSameTabNavigation` to `false`—but you’ll need to create a new Agent instance for each new tab. +If you want to restore opening in a new tab while keeping the Agent on the original page, set `forceSameTabNavigation` to `false` and create a new Agent instance for each new tab yourself. + +If subsequent actions should automatically continue in the newly opened tab, use `PlaywrightBrowserAgent` and enable `autoFollowNewPage`. ```typescript -const mid = new PlaywrightAgent(page, { - forceSameTabNavigation: false, +const mid = new PlaywrightBrowserAgent(context, page, { + autoFollowNewPage: true, }); ``` +Use `PlaywrightBrowserAgent.create(context, options)` only when you do not already have a page. The factory uses `initialPage` when provided, otherwise it reuses the first existing context page or creates a new page. + ### Connect Midscene Agent to a Remote Playwright Browser :::info Example Project diff --git a/apps/site/docs/en/integrate-with-puppeteer.mdx b/apps/site/docs/en/integrate-with-puppeteer.mdx index 8400efaff6..a36e0d3d34 100644 --- a/apps/site/docs/en/integrate-with-puppeteer.mdx +++ b/apps/site/docs/en/integrate-with-puppeteer.mdx @@ -106,14 +106,18 @@ After the above command executes successfully, the console will output: `Midscen Each Agent instance is bound to a single page. For easier debugging, Midscene intercepts new tabs by default (for example, links with `target="_blank"`) and opens them in the current page. -If you want to allow new tabs again, set `forceSameTabNavigation` to `false`—but you must create a new Agent instance for each new tab. +If you want to allow new tabs again while keeping the Agent on the original page, set `forceSameTabNavigation` to `false` and create a new Agent instance for each new tab yourself. + +If subsequent actions should automatically continue in the newly opened tab, use `PuppeteerBrowserAgent` and enable `autoFollowNewPage`. ```typescript -const mid = new PuppeteerAgent(page, { - forceSameTabNavigation: false, +const mid = new PuppeteerBrowserAgent(browser, page, { + autoFollowNewPage: true, }); ``` +Use `PuppeteerBrowserAgent.create(browser, options)` only when you do not already have a page. The factory uses `initialPage` when provided, otherwise it reuses the first existing browser page or creates a new page. + ### Connect Midscene Agent to a Remote Puppeteer Browser :::info Example Project diff --git a/apps/site/docs/en/web-api-reference.mdx b/apps/site/docs/en/web-api-reference.mdx index 55f4cf6397..c28c13ac82 100644 --- a/apps/site/docs/en/web-api-reference.mdx +++ b/apps/site/docs/en/web-api-reference.mdx @@ -56,11 +56,29 @@ In addition to the base agent options, Puppeteer exposes: :::info -- One agent per page: by default (`forceSameTabNavigation: true`), Midscene opens new links in the current tab for easier debugging. Set it to `false` if you want new tabs, and create a new agent per tab. +- One agent per page: by default (`forceSameTabNavigation: true`), Midscene opens new links in the current tab for easier debugging. Set it to `false` if you want normal new-tab behavior and create a new `PuppeteerAgent` for each page yourself. Use `PuppeteerBrowserAgent` when the same Agent should manage browser-level page switching. - For the full list of interaction methods, see [API reference (Common)](./api#interaction-methods). ::: +### Browser agent + +Use `PuppeteerBrowserAgent` when one Midscene Agent should manage page switching inside a Puppeteer browser. + +```ts +const agent = new PuppeteerBrowserAgent(browser, page, { + autoFollowNewPage: true, +}); +``` + +- Constructor: `new PuppeteerBrowserAgent(browser, page, options?)` — Use this when you already have the initial page. +- Factory: `PuppeteerBrowserAgent.create(browser, options?)` — Use this when you do not already have a page. It uses `initialPage` if provided, otherwise the first existing browser page, or creates a new page. +- `initialPage: Page` — Initial Puppeteer page for the factory. +- `autoFollowNewPage: boolean` — Automatically switch the active page when the browser opens a new page. Default `false`. +- `newPageTimeout: number` — Timeout for `waitForNewPage`. Default `5000`. +- `setActivePage(page: Page)` — Explicitly set which Puppeteer page the Browser Agent controls next. +- `waitForNewPage(action?, options?)` — Wait for a newly opened page without implicitly switching the active page. + ### Examples #### Quick start @@ -143,11 +161,29 @@ const agent = new PlaywrightAgent(page, { :::info -- One agent per page: with `forceSameTabNavigation` (default `true`), Midscene intercepts new tabs for stability. Set it to `false` to allow new tabs and create a separate agent for each. +- One agent per page: with `forceSameTabNavigation` (default `true`), Midscene intercepts new tabs for stability. Set it to `false` to allow normal new tabs and create a new `PlaywrightAgent` for each page yourself. Use `PlaywrightBrowserAgent` when the same Agent should manage browser-context-level page switching. - For the full list of interaction methods, see [API reference (Common)](./api#interaction-methods). ::: +### Browser agent + +Use `PlaywrightBrowserAgent` when one Midscene Agent should manage page switching inside a Playwright browser context. + +```ts +const agent = new PlaywrightBrowserAgent(context, page, { + autoFollowNewPage: true, +}); +``` + +- Constructor: `new PlaywrightBrowserAgent(context, page, options?)` — Use this when you already have the initial page. +- Factory: `PlaywrightBrowserAgent.create(context, options?)` — Use this when you do not already have a page. It uses `initialPage` if provided, otherwise the first existing context page, or creates a new page. +- `initialPage: Page` — Initial Playwright page for the factory. +- `autoFollowNewPage: boolean` — Automatically switch the active page when the context opens a new page. Default `false`. +- `newPageTimeout: number` — Timeout for `waitForNewPage`. Default `5000`. +- `setActivePage(page: Page)` — Explicitly set which Playwright page the Browser Agent controls next. +- `waitForNewPage(action?, options?)` — Wait for a newly opened page without implicitly switching the active page. + ### Examples #### Quick start diff --git a/apps/site/docs/zh/automate-with-scripts-in-yaml.mdx b/apps/site/docs/zh/automate-with-scripts-in-yaml.mdx index a7365a7425..f464d377dd 100644 --- a/apps/site/docs/zh/automate-with-scripts-in-yaml.mdx +++ b/apps/site/docs/zh/automate-with-scripts-in-yaml.mdx @@ -196,6 +196,10 @@ web: # 是否限制页面在当前 tab 打开，可选，默认 true forceSameTabNavigation: + # 是否使用 BrowserAgent 并自动继续在新打开的页面中执行，可选，默认 false + # 不能与 forceSameTabNavigation: true 同时使用 + autoFollowNewPage: + # CDP 连接端点，可选。设置后通过 CDP 连接到已有浏览器实例，而非启动新浏览器。 # 与 bridgeMode 互斥，不可同时使用。 cdpEndpoint: ws://localhost:9222/devtools/browser diff --git a/apps/site/docs/zh/integrate-with-playwright.mdx b/apps/site/docs/zh/integrate-with-playwright.mdx index 633e4d4538..fdbf7095d1 100644 --- a/apps/site/docs/zh/integrate-with-playwright.mdx +++ b/apps/site/docs/zh/integrate-with-playwright.mdx @@ -266,14 +266,18 @@ npx playwright test ./e2e/ebay-search.spec.ts 每个 Agent 实例都与对应的页面唯一绑定，为了方便开发者调试，Midscene 默认拦截了新 tab 的页面（如点击一个带有 `target="_blank"` 属性的链接），将其改为在当前页面打开。 -如果你想恢复在新标签页打开的行为，你可以设置 `forceSameTabNavigation` 选项为 `false`，但相应的，你需要为新标签页创建一个 Agent 实例。 +如果你想恢复在新标签页打开的行为，同时让当前 Agent 仍留在原页面，可以设置 `forceSameTabNavigation` 为 `false`，并自行为每个新标签页创建新的 Agent 实例。 + +如果后续操作要自动继续在新打开的标签页中执行，请使用 `PlaywrightBrowserAgent` 并开启 `autoFollowNewPage`。 ```typescript -const mid = new PlaywrightAgent(page, { - forceSameTabNavigation: false, +const mid = new PlaywrightBrowserAgent(context, page, { + autoFollowNewPage: true, }); ``` +仅当你手上还没有现成的 page 时，才需要使用 `PlaywrightBrowserAgent.create(context, options)`。这个工厂会优先使用 `initialPage`，否则复用 context 里的第一个页面，或者创建一个新页面。 + ### 连接远程 Playwright 浏览器并接入 Midscene Agent :::info 示例项目 diff --git a/apps/site/docs/zh/integrate-with-puppeteer.mdx b/apps/site/docs/zh/integrate-with-puppeteer.mdx index 8dbf103efd..7e65dd4f37 100644 --- a/apps/site/docs/zh/integrate-with-puppeteer.mdx +++ b/apps/site/docs/zh/integrate-with-puppeteer.mdx @@ -106,14 +106,18 @@ npx tsx demo.ts 每个 Agent 实例都与对应的页面唯一绑定，为了方便开发者调试，Midscene 默认拦截了新 tab 的页面（如点击一个带有 `target="_blank"` 属性的链接），将其改为在当前页面打开。 -如果你想恢复在新标签页打开的行为，你可以设置 `forceSameTabNavigation` 选项为 `false`，但相应的，你需要为新标签页创建一个 Agent 实例。 +如果你想恢复在新标签页打开的行为，同时让当前 Agent 仍留在原页面，可以设置 `forceSameTabNavigation` 为 `false`，并自行为每个新标签页创建新的 Agent 实例。 + +如果后续操作要自动继续在新打开的标签页中执行，请使用 `PuppeteerBrowserAgent` 并开启 `autoFollowNewPage`。 ```typescript -const mid = new PuppeteerAgent(page, { - forceSameTabNavigation: false, +const mid = new PuppeteerBrowserAgent(browser, page, { + autoFollowNewPage: true, }); ``` +仅当你手上还没有现成的 page 时，才需要使用 `PuppeteerBrowserAgent.create(browser, options)`。这个工厂会优先使用 `initialPage`，否则复用浏览器里的第一个页面，或者创建一个新页面。 + ### 连接远程 Puppeteer 浏览器并接入 Midscene Agent :::info 示例项目 diff --git a/apps/site/docs/zh/web-api-reference.mdx b/apps/site/docs/zh/web-api-reference.mdx index 6afe46ccfd..ce4e96adf2 100644 --- a/apps/site/docs/zh/web-api-reference.mdx +++ b/apps/site/docs/zh/web-api-reference.mdx @@ -56,11 +56,29 @@ const agent = new PuppeteerAgent(page, { :::info -- 每个页面一个 Agent：默认情况下（`forceSameTabNavigation: true`）Midscene 会拦截新标签并在当前页打开，便于调试；若想保留新标签行为可设为 `false`，并为每个标签创建新的 Agent。 +- 每个页面一个 Agent：默认情况下（`forceSameTabNavigation: true`）Midscene 会拦截新标签并在当前页打开，便于调试；若想保留浏览器原生的新标签行为可设为 `false`，并自行给每个页面创建新的 `PuppeteerAgent`。如果需要同一个 Agent 管理浏览器级别的页面切换，请使用 `PuppeteerBrowserAgent`。 - 更多交互方法请参考 [API 参考（通用）](./api#interaction-methods)。 ::: +### Browser Agent + +当一个 Midscene Agent 需要管理 Puppeteer 浏览器内的页面切换时，使用 `PuppeteerBrowserAgent`。 + +```ts +const agent = new PuppeteerBrowserAgent(browser, page, { + autoFollowNewPage: true, +}); +``` + +- 构造函数：`new PuppeteerBrowserAgent(browser, page, options?)` —— 已经有初始页面时使用。 +- 工厂方法：`PuppeteerBrowserAgent.create(browser, options?)` —— 手上还没有 page 时使用。它会优先使用 `initialPage`，否则复用浏览器里的第一个页面，或者创建一个新页面。 +- `initialPage: Page` —— 工厂方法使用的初始 Puppeteer 页面。 +- `autoFollowNewPage: boolean` —— 浏览器打开新页面时是否自动切换 active page，默认 `false`。 +- `newPageTimeout: number` —— `waitForNewPage` 的超时时间，默认 `5000`。 +- `setActivePage(page: Page)` —— 显式指定 Browser Agent 接下来控制哪个 Puppeteer 页面。 +- `waitForNewPage(action?, options?)` —— 等待新打开的页面，但不会隐式切换 active page。 + ### 示例 #### 快速上手 @@ -143,11 +161,29 @@ const agent = new PlaywrightAgent(page, { :::info -- 每个页面一个 Agent：默认 `forceSameTabNavigation` 为 `true`，Midscene 会拦截新标签确保稳定性；如需新标签请设为 `false` 并为每个标签创建新的 Agent。 +- 每个页面一个 Agent：默认 `forceSameTabNavigation` 为 `true`，Midscene 会拦截新标签确保稳定性；如需浏览器原生新标签行为请设为 `false`，并自行给每个页面创建新的 `PlaywrightAgent`。如果需要同一个 Agent 管理 browser context 级别的页面切换，请使用 `PlaywrightBrowserAgent`。 - 更多交互方法请参考 [API 参考（通用）](./api#interaction-methods)。 ::: +### Browser Agent + +当一个 Midscene Agent 需要管理 Playwright browser context 内的页面切换时，使用 `PlaywrightBrowserAgent`。 + +```ts +const agent = new PlaywrightBrowserAgent(context, page, { + autoFollowNewPage: true, +}); +``` + +- 构造函数：`new PlaywrightBrowserAgent(context, page, options?)` —— 已经有初始页面时使用。 +- 工厂方法：`PlaywrightBrowserAgent.create(context, options?)` —— 手上还没有 page 时使用。它会优先使用 `initialPage`，否则复用 context 里的第一个页面，或者创建一个新页面。 +- `initialPage: Page` —— 工厂方法使用的初始 Playwright 页面。 +- `autoFollowNewPage: boolean` —— context 打开新页面时是否自动切换 active page，默认 `false`。 +- `newPageTimeout: number` —— `waitForNewPage` 的超时时间，默认 `5000`。 +- `setActivePage(page: Page)` —— 显式指定 Browser Agent 接下来控制哪个 Playwright 页面。 +- `waitForNewPage(action?, options?)` —— 等待新打开的页面，但不会隐式切换 active page。 + ### 示例 #### 快速上手 diff --git a/packages/core/src/yaml.ts b/packages/core/src/yaml.ts index 52c8e6a878..dc4795ecaf 100644 --- a/packages/core/src/yaml.ts +++ b/packages/core/src/yaml.ts @@ -162,7 +162,8 @@ export interface MidsceneYamlScriptWebEnv */ extraHTTPHeaders?: Record; - forceSameTabNavigation?: boolean; // if track the newly opened tab, true for default in yaml script + forceSameTabNavigation?: boolean; // if limit the new tab to the current page, true for default in yaml script + autoFollowNewPage?: boolean; // if use BrowserAgent to follow newly opened pages, false for default /** * Chrome download directory (Puppeteer only, not supported in bridge mode). diff --git a/packages/web-integration/src/common/web-agent.ts b/packages/web-integration/src/common/web-agent.ts new file mode 100644 index 0000000000..3e40dc0037 --- /dev/null +++ b/packages/web-integration/src/common/web-agent.ts @@ -0,0 +1,72 @@ +import { getWebpackRequire } from '@/utils'; +import type { Page as PlaywrightPage } from 'playwright'; +import type { Page as PuppeteerPage } from 'puppeteer'; +import semver from 'semver'; +import { + BROWSER_NAVIGATION_ERROR_PATTERN, + forceChromeSelectRendering, +} from '../puppeteer/base-page'; + +type BrowserRuntime = 'puppeteer' | 'playwright'; + +const browserRuntimeConfig: Record< + BrowserRuntime, + { + displayName: string; + packageName: string; + minimumVersion: string; + requirementLabel: string; + } +> = { + puppeteer: { + displayName: 'Puppeteer', + packageName: 'puppeteer', + minimumVersion: '24.6.0', + requirementLabel: '> 24.6.0', + }, + playwright: { + displayName: 'Playwright', + packageName: 'playwright', + minimumVersion: '1.52.0', + requirementLabel: '>= 1.52.0', + }, +}; + +function getPackageVersion(packageName: string): string | null { + try { + const pkg = getWebpackRequire()(`${packageName}/package.json`); + return pkg.version || null; + } catch (error) { + console.error( + `[midscene:error] Failed to get ${packageName} version`, + error, + ); + return null; + } +} + +export function isRetryableBrowserNavigationError(error: unknown): boolean { + return ( + error instanceof Error && + BROWSER_NAVIGATION_ERROR_PATTERN.test(error.message) + ); +} + +export function applyForceChromeSelectRendering( + page: PuppeteerPage | PlaywrightPage, + runtime: BrowserRuntime, + enabled?: boolean, +): void { + if (!enabled) { + return; + } + + const config = browserRuntimeConfig[runtime]; + const version = getPackageVersion(config.packageName); + if (version && !semver.gte(version, config.minimumVersion)) { + console.warn( + `[midscene:error] forceChromeSelectRendering requires ${config.displayName} ${config.requirementLabel}, but current version is ${version}. This feature may not work correctly.`, + ); + } + forceChromeSelectRendering(page); +} diff --git a/packages/web-integration/src/index.ts b/packages/web-integration/src/index.ts index 0a75986f26..c571378ba3 100644 --- a/packages/web-integration/src/index.ts +++ b/packages/web-integration/src/index.ts @@ -2,8 +2,18 @@ export { PlaywrightAiFixture } from './playwright'; export type { PlayWrightAiFixtureType } from './playwright'; export { Agent as PageAgent, type AgentOpt } from '@midscene/core/agent'; -export { PuppeteerAgent } from './puppeteer'; -export { PlaywrightAgent } from './playwright'; +export { + PuppeteerAgent, + PuppeteerBrowserAgent, + type PuppeteerBrowserAgentCreateOpt, + type PuppeteerBrowserAgentOpt, +} from './puppeteer'; +export { + PlaywrightAgent, + PlaywrightBrowserAgent, + type PlaywrightBrowserAgentCreateOpt, + type PlaywrightBrowserAgentOpt, +} from './playwright'; export { StaticPageAgent, StaticPage } from './static'; export { WebMidsceneTools } from './agent-tools'; export { webPlaygroundPlatform } from './platform'; diff --git a/packages/web-integration/src/playwright/agent.ts b/packages/web-integration/src/playwright/agent.ts new file mode 100644 index 0000000000..dbb6bb2d72 --- /dev/null +++ b/packages/web-integration/src/playwright/agent.ts @@ -0,0 +1,41 @@ +import { + applyForceChromeSelectRendering, + isRetryableBrowserNavigationError, +} from '@/common/web-agent'; +import type { WebPageAgentOpt } from '@/web-element'; +import { Agent as PageAgent } from '@midscene/core/agent'; +import { getDebug } from '@midscene/shared/logger'; +import type { Page as PlaywrightPage } from 'playwright'; +import { forceClosePopup } from '../puppeteer/base-page'; +import { WebPage as PlaywrightWebPage } from './page'; + +const debug = getDebug('playwright:agent'); + +export class PlaywrightAgent extends PageAgent { + protected isRetryableContextError(error: unknown): boolean { + return isRetryableBrowserNavigationError(error); + } + + constructor(page: PlaywrightPage, opts?: WebPageAgentOpt) { + if (!page) { + throw new Error( + '[midscene] PlaywrightAgent requires a valid Playwright page instance. Please make sure to pass a valid page object.', + ); + } + const webPage = new PlaywrightWebPage(page, opts); + super(webPage, opts); + + const { forceSameTabNavigation = true, forceChromeSelectRendering } = + opts ?? {}; + + if (forceSameTabNavigation) { + forceClosePopup(page, debug); + } + + applyForceChromeSelectRendering( + page, + 'playwright', + forceChromeSelectRendering, + ); + } +} diff --git a/packages/web-integration/src/playwright/ai-fixture.ts b/packages/web-integration/src/playwright/ai-fixture.ts index 95ef92c20e..752431f175 100644 --- a/packages/web-integration/src/playwright/ai-fixture.ts +++ b/packages/web-integration/src/playwright/ai-fixture.ts @@ -1,4 +1,8 @@ -import { PlaywrightAgent, type PlaywrightWebPage } from '@/playwright/index'; +import { + PlaywrightAgent, + PlaywrightBrowserAgent, + type PlaywrightWebPage, +} from '@/playwright/index'; import type { WebPageAgentOpt } from '@/web-element'; import type { Cache } from '@midscene/core'; import type { Agent as PageAgent } from '@midscene/core/agent'; @@ -65,12 +69,14 @@ export type PlaywrightAiFixtureOptions = Omit< | 'reportFileName' | 'cache' > & { + autoFollowNewPage?: boolean; cache?: PlaywrightCache; }; export const PlaywrightAiFixture = (options?: PlaywrightAiFixtureOptions) => { const { forceSameTabNavigation = true, + autoFollowNewPage = false, waitForNetworkIdleTimeout = DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT, waitForNavigationTimeout = DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT, cache, @@ -143,17 +149,32 @@ export const PlaywrightAiFixture = (options?: PlaywrightAiFixtureOptions) => { // them here for the report tag only. const reportTag = `playwright-${title.replace(/[\\/]/g, '-')}-${idForPage}`; - const agent = new PlaywrightAgent(page, { + if (autoFollowNewPage && forceSameTabNavigation === true) { + throw new Error( + '[midscene] autoFollowNewPage cannot be used with forceSameTabNavigation: true.', + ); + } + + const commonAgentOpts = { testId: reportTag, reportFileName: reportTag, - forceSameTabNavigation, cache: cacheConfig, groupName: title, groupDescription: file, generateReport: true, ...sharedAgentOptions, ...opts, - }); + }; + + const agent = autoFollowNewPage + ? new PlaywrightBrowserAgent(page.context(), page, { + ...commonAgentOpts, + autoFollowNewPage: true, + }) + : new PlaywrightAgent(page, { + ...commonAgentOpts, + forceSameTabNavigation, + }); pageAgentMap[idForPage] = agent; const records = getAgentRecordsForTest(testInfo); const record: AgentRecord = { agent }; diff --git a/packages/web-integration/src/playwright/browser-agent.ts b/packages/web-integration/src/playwright/browser-agent.ts new file mode 100644 index 0000000000..afcd37d3e5 --- /dev/null +++ b/packages/web-integration/src/playwright/browser-agent.ts @@ -0,0 +1,193 @@ +import { + applyForceChromeSelectRendering, + isRetryableBrowserNavigationError, +} from '@/common/web-agent'; +import type { WebPageAgentOpt } from '@/web-element'; +import { Agent as PageAgent } from '@midscene/core/agent'; +import { getDebug } from '@midscene/shared/logger'; +import type { + BrowserContext as PlaywrightBrowserContext, + Page as PlaywrightPage, +} from 'playwright'; +import { WebPage as PlaywrightWebPage } from './page'; + +const debug = getDebug('playwright:browser-agent'); + +export type PlaywrightBrowserAgentOpt = Omit< + WebPageAgentOpt, + 'forceSameTabNavigation' +> & { + autoFollowNewPage?: boolean; + newPageTimeout?: number; +}; + +export type PlaywrightBrowserAgentCreateOpt = PlaywrightBrowserAgentOpt & { + initialPage?: PlaywrightPage; +}; + +export class PlaywrightBrowserAgent extends PageAgent { + private readonly context: PlaywrightBrowserContext; + private readonly autoFollowNewPage: boolean; + private readonly newPageTimeout: number; + + private readonly pageHandler = (page: PlaywrightPage) => { + void this.followPage(page); + }; + + protected isRetryableContextError(error: unknown): boolean { + return isRetryableBrowserNavigationError(error); + } + + constructor( + context: PlaywrightBrowserContext, + initialPage: PlaywrightPage, + opts?: PlaywrightBrowserAgentOpt, + ) { + if (!context) { + throw new Error( + '[midscene] PlaywrightBrowserAgent requires a valid Playwright browser context.', + ); + } + if (!initialPage) { + throw new Error( + '[midscene] PlaywrightBrowserAgent requires a valid initial page instance.', + ); + } + + const { + autoFollowNewPage = false, + newPageTimeout = 5000, + ...agentOpts + } = opts ?? {}; + const { forceChromeSelectRendering } = agentOpts; + const webPage = new PlaywrightWebPage(initialPage, { + ...agentOpts, + forceSameTabNavigation: false, + }); + super(webPage, agentOpts); + + this.context = context; + this.autoFollowNewPage = autoFollowNewPage; + this.newPageTimeout = newPageTimeout; + + if (this.autoFollowNewPage) { + this.context.on('page', this.pageHandler); + } + + applyForceChromeSelectRendering( + initialPage, + 'playwright', + forceChromeSelectRendering, + ); + } + + static async create( + context: PlaywrightBrowserContext, + opts?: PlaywrightBrowserAgentCreateOpt, + ) { + const { initialPage, ...agentOpts } = opts ?? {}; + const page = initialPage ?? context.pages()[0] ?? (await context.newPage()); + + return new PlaywrightBrowserAgent(context, page, agentOpts); + } + + get activePage() { + return this.interface.underlyingPage as PlaywrightPage; + } + + pages() { + return this.context.pages(); + } + + async newPage() { + const page = await this.context.newPage(); + await this.setActivePage(page); + return page; + } + + async setActivePage(page: PlaywrightPage) { + if (!page || page.isClosed()) { + throw new Error( + '[midscene] Cannot set PlaywrightBrowserAgent active page to a closed or invalid page.', + ); + } + + this.interface.underlyingPage = page; + try { + await page.bringToFront(); + } catch (error) { + debug(`failed to bring page to front: ${error}`); + } + } + + async waitForNewPage( + action?: () => Promise | unknown, + opts?: { timeout?: number }, + ) { + const waiter = this.createNewPageWaiter(opts?.timeout); + + try { + await action?.(); + return await waiter.promise; + } catch (error) { + waiter.dispose(); + throw error; + } + } + + async destroy() { + this.context.off('page', this.pageHandler); + await super.destroy(); + } + + private async followPage(page: PlaywrightPage) { + try { + await this.setActivePage(page); + } catch (error) { + debug(`failed to follow new page: ${error}`); + } + } + + private createNewPageWaiter(timeout = this.newPageTimeout) { + let settled = false; + + const dispose = () => { + this.context.off('page', handler); + clearTimeout(timer); + }; + + const handler = (page: PlaywrightPage) => { + if (settled) { + return; + } + + settled = true; + dispose(); + resolvePage(page); + }; + + let resolvePage!: (page: PlaywrightPage) => void; + let rejectPage!: (error: unknown) => void; + const promise = new Promise((resolve, reject) => { + resolvePage = resolve; + rejectPage = reject; + }); + + const timer = setTimeout(() => { + if (settled) { + return; + } + settled = true; + dispose(); + rejectPage( + new Error( + `[midscene] Timed out waiting for a new Playwright page after ${timeout}ms.`, + ), + ); + }, timeout); + + this.context.on('page', handler); + + return { promise, dispose }; + } +} diff --git a/packages/web-integration/src/playwright/index.ts b/packages/web-integration/src/playwright/index.ts index 02c51d6172..da45d28992 100644 --- a/packages/web-integration/src/playwright/index.ts +++ b/packages/web-integration/src/playwright/index.ts @@ -1,7 +1,3 @@ -import { Agent as PageAgent } from '@midscene/core/agent'; -import type { Page as PlaywrightPage } from 'playwright'; -import { WebPage as PlaywrightWebPage } from './page'; - export type { PlayWrightAiFixtureType, PlaywrightAiFixtureOptions, @@ -10,67 +6,9 @@ export { PlaywrightAiFixture } from './ai-fixture'; export { overrideAIConfig } from '@midscene/shared/env'; export { WebPage as PlaywrightWebPage } from './page'; export type { WebPageAgentOpt } from '@/web-element'; -import type { WebPageAgentOpt } from '@/web-element'; -import { getDebug } from '@midscene/shared/logger'; -import semver from 'semver'; -import { - BROWSER_NAVIGATION_ERROR_PATTERN, - autoSwitchToNewTab, - forceChromeSelectRendering as applyChromeSelectRendering, - forceClosePopup, -} from '../puppeteer/base-page'; -import { getWebpackRequire } from '../utils'; - -const debug = getDebug('playwright:agent'); - -/** - * Get Playwright version from package.json - */ -function getPlaywrightVersion(): string | null { - try { - const playwrightPkg = getWebpackRequire()('playwright/package.json'); - return playwrightPkg.version || null; - } catch (error) { - console.error('[midscene:error] Failed to get Playwright version', error); - return null; - } -} - -export class PlaywrightAgent extends PageAgent { - protected isRetryableContextError(error: unknown): boolean { - return ( - error instanceof Error && - BROWSER_NAVIGATION_ERROR_PATTERN.test(error.message) - ); - } - - constructor(page: PlaywrightPage, opts?: WebPageAgentOpt) { - if (!page) { - throw new Error( - '[midscene] PlaywrightAgent requires a valid Playwright page instance. Please make sure to pass a valid page object.', - ); - } - const webPage = new PlaywrightWebPage(page, opts); - super(webPage, opts); - - const { forceSameTabNavigation = true, forceChromeSelectRendering } = - opts ?? {}; - - if (forceSameTabNavigation) { - forceClosePopup(page, debug); - } else { - autoSwitchToNewTab(page, webPage, debug); - } - - if (forceChromeSelectRendering) { - // Check Playwright version requirement (>= 1.52) - const playwrightVersion = getPlaywrightVersion(); - if (playwrightVersion && !semver.gte(playwrightVersion, '1.52.0')) { - console.warn( - `[midscene:error] forceChromeSelectRendering requires Playwright >= 1.52.0, but current version is ${playwrightVersion}. This feature may not work correctly.`, - ); - } - applyChromeSelectRendering(page); - } - } -} +export { PlaywrightAgent } from './agent'; +export { + PlaywrightBrowserAgent, + type PlaywrightBrowserAgentCreateOpt, + type PlaywrightBrowserAgentOpt, +} from './browser-agent'; diff --git a/packages/web-integration/src/puppeteer/agent-launcher.ts b/packages/web-integration/src/puppeteer/agent-launcher.ts index b7810ca82c..200716cda7 100644 --- a/packages/web-integration/src/puppeteer/agent-launcher.ts +++ b/packages/web-integration/src/puppeteer/agent-launcher.ts @@ -8,7 +8,7 @@ import { defaultViewportWidth, resolveWebViewportSize, } from '@/common/viewport'; -import { PuppeteerAgent } from '@/puppeteer/index'; +import { PuppeteerAgent, PuppeteerBrowserAgent } from '@/puppeteer/index'; import type { AgentOpt, Cache, MidsceneYamlScriptWebEnv } from '@midscene/core'; import { DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT } from '@midscene/shared/constants'; import puppeteer, { @@ -385,19 +385,37 @@ export async function puppeteerAgentForTarget( const { aiActionContext, ...preferenceToUse } = preference ?? {}; - // prepare Midscene agent - const agent = new PuppeteerAgent(page, { + const forceSameTabNavigation = + typeof target.forceSameTabNavigation !== 'undefined' + ? target.forceSameTabNavigation + : true; + + if (target.autoFollowNewPage && target.forceSameTabNavigation === true) { + throw new Error( + '[midscene] autoFollowNewPage cannot be used with forceSameTabNavigation: true.', + ); + } + + const commonAgentOpts = { ...preferenceToUse, aiActContext, waitForNetworkIdleTimeout: typeof target.waitForNetworkIdle?.timeout === 'number' ? target.waitForNetworkIdle.timeout : undefined, - forceSameTabNavigation: - typeof target.forceSameTabNavigation !== 'undefined' - ? target.forceSameTabNavigation - : true, // true for default in yaml script - }); + }; + + // prepare Midscene agent + const agent = target.autoFollowNewPage + ? await PuppeteerBrowserAgent.create(page.browser(), { + ...commonAgentOpts, + initialPage: page, + autoFollowNewPage: true, + }) + : new PuppeteerAgent(page, { + ...commonAgentOpts, + forceSameTabNavigation, + }); freeFn.push({ name: 'midscene_puppeteer_agent', diff --git a/packages/web-integration/src/puppeteer/agent.ts b/packages/web-integration/src/puppeteer/agent.ts new file mode 100644 index 0000000000..3a5685ea16 --- /dev/null +++ b/packages/web-integration/src/puppeteer/agent.ts @@ -0,0 +1,41 @@ +import { + applyForceChromeSelectRendering, + isRetryableBrowserNavigationError, +} from '@/common/web-agent'; +import type { WebPageAgentOpt } from '@/web-element'; +import { Agent as PageAgent } from '@midscene/core/agent'; +import { getDebug } from '@midscene/shared/logger'; +import type { Page as PuppeteerPage } from 'puppeteer'; +import { forceClosePopup } from './base-page'; +import { PuppeteerWebPage } from './page'; + +const debug = getDebug('puppeteer:agent'); + +export class PuppeteerAgent extends PageAgent { + protected isRetryableContextError(error: unknown): boolean { + return isRetryableBrowserNavigationError(error); + } + + constructor(page: PuppeteerPage, opts?: WebPageAgentOpt) { + if (!page) { + throw new Error( + '[midscene] PuppeteerAgent requires a valid Puppeteer page instance. Please make sure to pass a valid page object.', + ); + } + const webPage = new PuppeteerWebPage(page, opts); + super(webPage, opts); + + const { forceSameTabNavigation = true, forceChromeSelectRendering } = + opts ?? {}; + + if (forceSameTabNavigation) { + forceClosePopup(page, debug); + } + + applyForceChromeSelectRendering( + page, + 'puppeteer', + forceChromeSelectRendering, + ); + } +} diff --git a/packages/web-integration/src/puppeteer/base-page.ts b/packages/web-integration/src/puppeteer/base-page.ts index 866a0b6a4c..f12b4c6ffc 100644 --- a/packages/web-integration/src/puppeteer/base-page.ts +++ b/packages/web-integration/src/puppeteer/base-page.ts @@ -1265,55 +1265,6 @@ export function forceClosePopup( }); } -/** - * When forceSameTabNavigation is false, automatically switch the agent's - * underlying page reference to newly opened tabs so that subsequent actions - * operate on the new tab instead of the original one. - */ -export function autoSwitchToNewTab( - page: PuppeteerPage | PlaywrightPage, - webPage: Page, - debugProfile: DebugFunction, -) { - page.on('popup', async (popup) => { - if (!popup) { - console.warn( - '[midscene] got a popup event, but the popup is not ready yet, skip', - ); - return; - } - - try { - const popupPage = popup as PuppeteerPage; - if (popupPage.isClosed()) { - debugProfile('popup is already closed, skip switching'); - return; - } - - const url = popupPage.url(); - debugProfile(`New tab detected: ${url}, switching to it`); - - // Update the underlying page reference to the new tab - webPage.underlyingPage = popup as any; - - // Bring the new tab to the front - await popupPage.bringToFront(); - - // Recursively listen for popups on the new tab as well - autoSwitchToNewTab( - popup as PuppeteerPage | PlaywrightPage, - webPage, - debugProfile, - ); - } catch (error) { - debugProfile(`failed to switch to new tab: ${error}`); - console.warn( - `[midscene:warning] Failed to switch to newly opened tab: ${error}`, - ); - } - }); -} - /** * Force Chrome to render select elements using base-select appearance instead of OS-native rendering. * This makes select elements visible in screenshots captured by Playwright/Puppeteer. diff --git a/packages/web-integration/src/puppeteer/browser-agent.ts b/packages/web-integration/src/puppeteer/browser-agent.ts new file mode 100644 index 0000000000..5f4e72d3d3 --- /dev/null +++ b/packages/web-integration/src/puppeteer/browser-agent.ts @@ -0,0 +1,211 @@ +import { + applyForceChromeSelectRendering, + isRetryableBrowserNavigationError, +} from '@/common/web-agent'; +import type { WebPageAgentOpt } from '@/web-element'; +import { Agent as PageAgent } from '@midscene/core/agent'; +import { getDebug } from '@midscene/shared/logger'; +import type { + Browser as PuppeteerBrowser, + Page as PuppeteerPage, + Target as PuppeteerTarget, +} from 'puppeteer'; +import { PuppeteerWebPage } from './page'; + +const debug = getDebug('puppeteer:browser-agent'); + +export type PuppeteerBrowserAgentOpt = Omit< + WebPageAgentOpt, + 'forceSameTabNavigation' +> & { + autoFollowNewPage?: boolean; + newPageTimeout?: number; +}; + +export type PuppeteerBrowserAgentCreateOpt = PuppeteerBrowserAgentOpt & { + initialPage?: PuppeteerPage; +}; + +export class PuppeteerBrowserAgent extends PageAgent { + private readonly browser: PuppeteerBrowser; + private readonly autoFollowNewPage: boolean; + private readonly newPageTimeout: number; + + private readonly targetCreatedHandler = (target: PuppeteerTarget) => { + void this.followTarget(target); + }; + + protected isRetryableContextError(error: unknown): boolean { + return isRetryableBrowserNavigationError(error); + } + + constructor( + browser: PuppeteerBrowser, + initialPage: PuppeteerPage, + opts?: PuppeteerBrowserAgentOpt, + ) { + if (!browser) { + throw new Error( + '[midscene] PuppeteerBrowserAgent requires a valid Puppeteer browser instance.', + ); + } + if (!initialPage) { + throw new Error( + '[midscene] PuppeteerBrowserAgent requires a valid initial page instance.', + ); + } + + const { + autoFollowNewPage = false, + newPageTimeout = 5000, + ...agentOpts + } = opts ?? {}; + const { forceChromeSelectRendering } = agentOpts; + const webPage = new PuppeteerWebPage(initialPage, { + ...agentOpts, + forceSameTabNavigation: false, + }); + super(webPage, agentOpts); + + this.browser = browser; + this.autoFollowNewPage = autoFollowNewPage; + this.newPageTimeout = newPageTimeout; + + if (this.autoFollowNewPage) { + this.browser.on('targetcreated', this.targetCreatedHandler); + } + + applyForceChromeSelectRendering( + initialPage, + 'puppeteer', + forceChromeSelectRendering, + ); + } + + static async create( + browser: PuppeteerBrowser, + opts?: PuppeteerBrowserAgentCreateOpt, + ) { + const { initialPage, ...agentOpts } = opts ?? {}; + const page = + initialPage ?? (await browser.pages())[0] ?? (await browser.newPage()); + + return new PuppeteerBrowserAgent(browser, page, agentOpts); + } + + get activePage() { + return this.interface.underlyingPage as PuppeteerPage; + } + + async pages() { + return this.browser.pages(); + } + + async newPage() { + const page = await this.browser.newPage(); + await this.setActivePage(page); + return page; + } + + async setActivePage(page: PuppeteerPage) { + if (!page || page.isClosed()) { + throw new Error( + '[midscene] Cannot set PuppeteerBrowserAgent active page to a closed or invalid page.', + ); + } + + this.interface.underlyingPage = page; + try { + await page.bringToFront(); + } catch (error) { + debug(`failed to bring page to front: ${error}`); + } + } + + async waitForNewPage( + action?: () => Promise | unknown, + opts?: { timeout?: number }, + ) { + const waiter = this.createNewPageWaiter(opts?.timeout); + + try { + await action?.(); + return await waiter.promise; + } catch (error) { + waiter.dispose(); + throw error; + } + } + + async destroy() { + this.browser.off('targetcreated', this.targetCreatedHandler); + await super.destroy(); + } + + private async followTarget(target: PuppeteerTarget) { + if (target.type() !== 'page') { + return; + } + + try { + const page = await target.page(); + if (page) { + await this.setActivePage(page); + } + } catch (error) { + debug(`failed to follow new page: ${error}`); + } + } + + private createNewPageWaiter(timeout = this.newPageTimeout) { + let settled = false; + + const dispose = () => { + this.browser.off('targetcreated', handler); + clearTimeout(timer); + }; + + const handler = async (target: PuppeteerTarget) => { + if (target.type() !== 'page' || settled) { + return; + } + + settled = true; + dispose(); + + try { + const page = await target.page(); + if (!page) { + throw new Error('new target did not resolve to a page'); + } + resolvePage(page); + } catch (error) { + rejectPage(error); + } + }; + + let resolvePage!: (page: PuppeteerPage) => void; + let rejectPage!: (error: unknown) => void; + const promise = new Promise((resolve, reject) => { + resolvePage = resolve; + rejectPage = reject; + }); + + const timer = setTimeout(() => { + if (settled) { + return; + } + settled = true; + dispose(); + rejectPage( + new Error( + `[midscene] Timed out waiting for a new Puppeteer page after ${timeout}ms.`, + ), + ); + }, timeout); + + this.browser.on('targetcreated', handler); + + return { promise, dispose }; + } +} diff --git a/packages/web-integration/src/puppeteer/index.ts b/packages/web-integration/src/puppeteer/index.ts index 440846cf37..0f0b662a9b 100644 --- a/packages/web-integration/src/puppeteer/index.ts +++ b/packages/web-integration/src/puppeteer/index.ts @@ -1,73 +1,11 @@ -import type { WebPageAgentOpt } from '@/web-element'; -import { Agent as PageAgent } from '@midscene/core/agent'; -import { getDebug } from '@midscene/shared/logger'; -import type { Page as PuppeteerPage } from 'puppeteer'; -import semver from 'semver'; -import { getWebpackRequire } from '../utils'; -import { - BROWSER_NAVIGATION_ERROR_PATTERN, - autoSwitchToNewTab, - forceChromeSelectRendering as applyChromeSelectRendering, - forceClosePopup, -} from './base-page'; -import { PuppeteerWebPage } from './page'; - -const debug = getDebug('puppeteer:agent'); - -/** - * Get Puppeteer version from package.json - */ -function getPuppeteerVersion(): string | null { - try { - const puppeteerPkg = getWebpackRequire()('puppeteer/package.json'); - return puppeteerPkg.version || null; - } catch (error) { - console.error('[midscene:error] Failed to get Puppeteer version', error); - return null; - } -} - +export { PuppeteerAgent } from './agent'; +export { + PuppeteerBrowserAgent, + type PuppeteerBrowserAgentCreateOpt, + type PuppeteerBrowserAgentOpt, +} from './browser-agent'; export { PuppeteerWebPage } from './page'; export type { WebPageAgentOpt } from '@/web-element'; - -export class PuppeteerAgent extends PageAgent { - protected isRetryableContextError(error: unknown): boolean { - return ( - error instanceof Error && - BROWSER_NAVIGATION_ERROR_PATTERN.test(error.message) - ); - } - - constructor(page: PuppeteerPage, opts?: WebPageAgentOpt) { - if (!page) { - throw new Error( - '[midscene] PuppeteerAgent requires a valid Puppeteer page instance. Please make sure to pass a valid page object.', - ); - } - const webPage = new PuppeteerWebPage(page, opts); - super(webPage, opts); - - const { forceSameTabNavigation = true, forceChromeSelectRendering } = - opts ?? {}; - - if (forceSameTabNavigation) { - forceClosePopup(page, debug); - } else { - autoSwitchToNewTab(page, webPage, debug); - } - - if (forceChromeSelectRendering) { - const puppeteerVersion = getPuppeteerVersion(); - if (puppeteerVersion && !semver.gte(puppeteerVersion, '24.6.0')) { - console.warn( - `[midscene:error] forceChromeSelectRendering requires Puppeteer > 24.6.0, but current version is ${puppeteerVersion}. This feature may not work correctly.`, - ); - } - applyChromeSelectRendering(page); - } - } -} - export { overrideAIConfig } from '@midscene/shared/env'; // Do NOT export this since it requires puppeteer diff --git a/packages/web-integration/tests/ai/web/puppeteer/tab-navigation.test.ts b/packages/web-integration/tests/ai/web/puppeteer/tab-navigation.test.ts index ff3967f8bf..dbf8ed02f2 100644 --- a/packages/web-integration/tests/ai/web/puppeteer/tab-navigation.test.ts +++ b/packages/web-integration/tests/ai/web/puppeteer/tab-navigation.test.ts @@ -1,5 +1,6 @@ -import { PuppeteerAgent } from '@/puppeteer'; +import { PuppeteerAgent, PuppeteerBrowserAgent } from '@/puppeteer'; import { sleep } from '@midscene/core/utils'; +import type { Page as PuppeteerPage } from 'puppeteer'; import { describe, expect, it } from 'vitest'; import { DEFAULT_TEST_TIMEOUT, @@ -8,41 +9,128 @@ import { } from './test-utils'; import { launchPage } from './utils'; +const clickNewTabLink = async (page: PuppeteerPage) => { + const popupPromise = new Promise((resolve) => { + page.once('popup', resolve); + }); + + await page.click('#newTabLink'); + const popup = await popupPromise; + await popup.waitForSelector('.weather-container'); + return popup; +}; + +const tapNewTabLinkWithAgent = async ( + agent: PuppeteerAgent | PuppeteerBrowserAgent, + page: PuppeteerPage, +) => { + const popupPromise = new Promise((resolve) => { + page.once('popup', resolve); + }); + + await agent.aiTap('the "Open in New Tab" link on the original page', { + xpath: '//*[@id="newTabLink"]', + }); + + const popup = await popupPromise; + await popup.waitForSelector('.weather-container'); + return popup; +}; + +const waitForAgentPage = async ( + agent: PuppeteerAgent | PuppeteerBrowserAgent, + expectedPage: PuppeteerPage, +) => { + for (let i = 0; i < 20; i++) { + if (agent.page.underlyingPage === expectedPage) { + return; + } + await sleep(100); + } + + throw new Error('Timed out waiting for the agent to switch tabs'); +}; + describe( 'Tab Navigation Tests', () => { const ctx = createTestContext(); - it('auto switch to new tab', async () => { + it('keeps page agent on current page when browser agent is not used', async () => { const htmlPath = getFixturePath('tab-navigation.html'); const { originPage, reset } = await launchPage(`file://${htmlPath}`); ctx.resetFn = reset; ctx.agent = new PuppeteerAgent(originPage, { forceSameTabNavigation: false, }); - await ctx.agent.aiTap('the "Open in New Tab" link', { - deepThink: true, + + const popup = await clickNewTabLink(originPage); + + expect(ctx.agent.page.underlyingPage).toBe(originPage); + expect(originPage.url()).toContain('tab-navigation.html'); + expect(popup.url()).toContain('tab-navigation-target.html'); + }); + + it('switches to a new page when controlled manually by browser agent', async () => { + const htmlPath = getFixturePath('tab-navigation.html'); + const { originPage, reset } = await launchPage(`file://${htmlPath}`); + ctx.resetFn = reset; + ctx.agent = new PuppeteerBrowserAgent(originPage.browser(), originPage); + + const popup = await ctx.agent.waitForNewPage(() => + originPage.click('#newTabLink'), + ); + await popup.waitForSelector('.weather-container'); + + expect(ctx.agent.page.underlyingPage).toBe(originPage); + + await ctx.agent.setActivePage(popup); + + expect(ctx.agent.page.underlyingPage).toBe(popup); + await expect(ctx.agent.page.url()).resolves.toContain( + 'tab-navigation-target.html', + ); + }); + + it('auto follows new page when browser agent option is enabled', async () => { + const htmlPath = getFixturePath('tab-navigation.html'); + const { originPage, reset } = await launchPage(`file://${htmlPath}`); + ctx.resetFn = reset; + ctx.agent = new PuppeteerBrowserAgent(originPage.browser(), originPage, { + autoFollowNewPage: true, }); - await sleep(3000); - // When forceSameTabNavigation is false, the agent should automatically - // switch to the newly opened tab so subsequent actions operate on it - await ctx.agent.aiWaitFor('There is a weather forecast in the page'); + const popup = await tapNewTabLinkWithAgent(ctx.agent, originPage); + await waitForAgentPage(ctx.agent, popup); + + expect(ctx.agent.page.underlyingPage).toBe(popup); + await expect(ctx.agent.page.url()).resolves.toContain( + 'tab-navigation-target.html', + ); + + await ctx.agent.aiTap( + 'the "Weather Forecast" heading in the newly opened tab', + { + xpath: '//h2[text()="Weather Forecast"]', + }, + ); + + expect(ctx.agent.page.underlyingPage).toBe(popup); }); - it('tracking active tab', async () => { + it('keeps same-tab navigation behavior for page agent', async () => { const htmlPath = getFixturePath('tab-navigation.html'); const { originPage, reset } = await launchPage(`file://${htmlPath}`); ctx.resetFn = reset; ctx.agent = new PuppeteerAgent(originPage, { forceSameTabNavigation: true, }); - await ctx.agent.aiTap('the "Open in New Tab" link', { - deepThink: true, - }); - // When forceSameTabNavigation is true, the agent should follow the new tab - await ctx.agent.aiWaitFor('There is a weather forecast in the page'); + await originPage.click('#newTabLink'); + await originPage.waitForSelector('.weather-container'); + + expect(ctx.agent.page.underlyingPage).toBe(originPage); + expect(originPage.url()).toContain('tab-navigation-target.html'); }); }, DEFAULT_TEST_TIMEOUT, diff --git a/packages/web-integration/tests/ai/web/puppeteer/test-utils.ts b/packages/web-integration/tests/ai/web/puppeteer/test-utils.ts index f141a93bde..b5f6ede63d 100644 --- a/packages/web-integration/tests/ai/web/puppeteer/test-utils.ts +++ b/packages/web-integration/tests/ai/web/puppeteer/test-utils.ts @@ -1,5 +1,5 @@ import path from 'node:path'; -import type { PuppeteerAgent } from '@/puppeteer'; +import type { PuppeteerAgent, PuppeteerBrowserAgent } from '@/puppeteer'; import { afterEach } from 'vitest'; /** @@ -18,7 +18,7 @@ export function getFixturePath(filename: string): string { * Shared test context for Puppeteer integration tests */ export interface TestContext { - agent: PuppeteerAgent | null; + agent: PuppeteerAgent | PuppeteerBrowserAgent | null; resetFn: (() => Promise) | null; } From 2b75efa01c2455cea436e109f4c4a3cf163a0cef Mon Sep 17 00:00:00 2001 From: quanru Date: Thu, 11 Jun 2026 11:53:43 +0800 Subject: [PATCH 3/7] fix(web-integration): split page and browser agent targets --- .../docs/en/automate-with-scripts-in-yaml.mdx | 40 ++++- .../docs/en/integrate-with-playwright.mdx | 6 +- .../site/docs/en/integrate-with-puppeteer.mdx | 6 +- apps/site/docs/en/web-api-reference.mdx | 34 ++-- apps/site/docs/en/yaml-script-runner.mdx | 14 +- .../docs/zh/automate-with-scripts-in-yaml.mdx | 41 ++++- .../docs/zh/integrate-with-playwright.mdx | 6 +- .../site/docs/zh/integrate-with-puppeteer.mdx | 6 +- apps/site/docs/zh/web-api-reference.mdx | 34 ++-- apps/site/docs/zh/yaml-script-runner.mdx | 16 +- packages/cli/src/config-factory.ts | 20 ++- packages/cli/src/create-yaml-player.ts | 27 ++- packages/cli/src/framework/yaml-case.ts | 2 + packages/cli/src/yaml-batch-executor.ts | 35 ++-- .../tests/unit-test/config-factory.test.ts | 52 ++++++ .../unit-test/create-yaml-player.test.ts | 99 +++++++++++ packages/core/src/yaml.ts | 8 + packages/core/src/yaml/builder.ts | 2 +- packages/core/src/yaml/utils.ts | 104 +++++++++++ .../src/common/browser-agent.ts | 167 ++++++++++++++++++ packages/web-integration/src/index.ts | 2 + .../web-integration/src/playwright/agent.ts | 45 +---- .../src/playwright/browser-agent.ts | 129 ++++---------- .../web-integration/src/playwright/index.ts | 2 +- .../src/playwright/page-agent.ts | 43 +++++ .../src/puppeteer/agent-launcher.ts | 41 +++-- .../web-integration/src/puppeteer/agent.ts | 45 +---- .../src/puppeteer/browser-agent.ts | 148 ++++------------ .../web-integration/src/puppeteer/index.ts | 2 +- .../src/puppeteer/page-agent.ts | 43 +++++ .../browser-agent-page-controller.test.ts | 122 +++++++++++++ .../unit-test/circular-dependency.test.ts | 12 +- .../unit-test/constructor-validation.test.ts | 20 ++- .../puppeteer/agent-launcher.test.ts | 37 ++++ .../yaml/__snapshots__/utils.test.ts.snap | 2 +- .../tests/unit-test/yaml/utils.test.ts | 99 ++++++++++- 36 files changed, 1112 insertions(+), 399 deletions(-) create mode 100644 packages/web-integration/src/common/browser-agent.ts create mode 100644 packages/web-integration/src/playwright/page-agent.ts create mode 100644 packages/web-integration/src/puppeteer/page-agent.ts create mode 100644 packages/web-integration/tests/unit-test/browser-agent-page-controller.test.ts diff --git a/apps/site/docs/en/automate-with-scripts-in-yaml.mdx b/apps/site/docs/en/automate-with-scripts-in-yaml.mdx index 2e362821bf..378cf6a061 100644 --- a/apps/site/docs/en/automate-with-scripts-in-yaml.mdx +++ b/apps/site/docs/en/automate-with-scripts-in-yaml.mdx @@ -9,7 +9,7 @@ Midscene offers a way to perform automation using `.yaml` files, which helps you Here is an example. By reading its content, you should be able to understand how it works. ```yaml -web: +page: url: https://www.bing.com tasks: @@ -41,10 +41,10 @@ To execute YAML workflows from the command line, install the Midscene CLI. See [ Script files use YAML format to describe automation tasks. It defines the target to be manipulated (like a webpage or an Android app) and the series of steps to perform. -A standard `.yaml` script file includes a `web`, `android`, `ios`, `harmony`, or `computer` section to configure the environment, an optional `agent` section to configure AI agent behavior, and a `tasks` section to define the automation tasks. +A standard `.yaml` script file includes a `page`, `browser`, `web`, `android`, `ios`, `harmony`, or `computer` section to configure the environment, an optional `agent` section to configure AI agent behavior, and a `tasks` section to define the automation tasks. ```yaml -web: +page: url: https://www.bing.com # The tasks section defines the series of steps to be executed @@ -56,6 +56,7 @@ tasks: - aiAssert: The results show weather information ``` +Use `page:` for a page-level Agent. Use `browser:` when one Agent should manage a browser and its active page. `web:` remains supported as a compatibility entry; `web.mode: browser` maps to BrowserAgent and plain `web:` maps to PageAgent. Do not combine `page`, `browser`, `web`, or the deprecated `target` in the same script. ### The `agent` part @@ -141,10 +142,36 @@ tasks: ``` -### The `web` part +### The web target part + +Recommended page-level target: + +```yaml +page: + url: https://example.com +``` + +Recommended browser-level target: + +```yaml +browser: + url: https://example.com + autoFollowNewPage: true +``` + +Compatibility form: ```yaml web: + mode: browser + url: https://example.com + autoFollowNewPage: true +``` + +Shared options: + +```yaml +page: # The URL to visit, required. If `serve` is provided, provide the relative path. url: @@ -195,10 +222,11 @@ web: unstableLogContent: # Whether to restrict page navigation to the current tab, optional, defaults to true. + # Page mode only. Do not use it with `browser:` or `web.mode: browser`. forceSameTabNavigation: - # Whether to use BrowserAgent and automatically continue in newly opened pages, optional, defaults to false. - # Cannot be used together with forceSameTabNavigation: true. + # Whether BrowserAgent should automatically continue in newly opened pages, optional, defaults to false. + # Browser mode only. Use `browser:` or `web.mode: browser`. autoFollowNewPage: # CDP endpoint, optional. Connects to an existing browser instance via CDP instead of launching a new one. diff --git a/apps/site/docs/en/integrate-with-playwright.mdx b/apps/site/docs/en/integrate-with-playwright.mdx index e412d4f4b2..121e03210e 100644 --- a/apps/site/docs/en/integrate-with-playwright.mdx +++ b/apps/site/docs/en/integrate-with-playwright.mdx @@ -265,11 +265,11 @@ After the command executes successfully, it will output: `Midscene - report file ### About opening in a new tab -Each Agent instance is bound to a single page. To make debugging easier, Midscene intercepts new tabs by default (for example, links with `target="_blank"`) and opens them in the current page. +`PlaywrightAgent` is a page-level Agent: each instance is bound to a single page. To make debugging easier, Midscene intercepts new tabs by default (for example, links with `target="_blank"`) and opens them in the current page. If you want to restore opening in a new tab while keeping the Agent on the original page, set `forceSameTabNavigation` to `false` and create a new Agent instance for each new tab yourself. -If subsequent actions should automatically continue in the newly opened tab, use `PlaywrightBrowserAgent` and enable `autoFollowNewPage`. +If one Agent should manage page switching for the whole browser context, use `PlaywrightBrowserAgent`. Enable `autoFollowNewPage` when subsequent actions should automatically continue in the newly opened tab. ```typescript const mid = new PlaywrightBrowserAgent(context, page, { @@ -277,7 +277,7 @@ const mid = new PlaywrightBrowserAgent(context, page, { }); ``` -Use `PlaywrightBrowserAgent.create(context, options)` only when you do not already have a page. The factory uses `initialPage` when provided, otherwise it reuses the first existing context page or creates a new page. +Use `new PlaywrightBrowserAgent(context, page, options)` when you explicitly choose the initial active page. Use `PlaywrightBrowserAgent.create(context, options)` when you want Midscene to choose or create the initial active page; the factory uses `initialPage` when provided, otherwise it reuses the first existing context page or creates a new page. ### Connect Midscene Agent to a Remote Playwright Browser diff --git a/apps/site/docs/en/integrate-with-puppeteer.mdx b/apps/site/docs/en/integrate-with-puppeteer.mdx index a36e0d3d34..cd742e1c09 100644 --- a/apps/site/docs/en/integrate-with-puppeteer.mdx +++ b/apps/site/docs/en/integrate-with-puppeteer.mdx @@ -104,11 +104,11 @@ After the above command executes successfully, the console will output: `Midscen ### About opening in a new tab -Each Agent instance is bound to a single page. For easier debugging, Midscene intercepts new tabs by default (for example, links with `target="_blank"`) and opens them in the current page. +`PuppeteerAgent` is a page-level Agent: each instance is bound to a single page. For easier debugging, Midscene intercepts new tabs by default (for example, links with `target="_blank"`) and opens them in the current page. If you want to allow new tabs again while keeping the Agent on the original page, set `forceSameTabNavigation` to `false` and create a new Agent instance for each new tab yourself. -If subsequent actions should automatically continue in the newly opened tab, use `PuppeteerBrowserAgent` and enable `autoFollowNewPage`. +If one Agent should manage page switching for the whole browser, use `PuppeteerBrowserAgent`. Enable `autoFollowNewPage` when subsequent actions should automatically continue in the newly opened tab. ```typescript const mid = new PuppeteerBrowserAgent(browser, page, { @@ -116,7 +116,7 @@ const mid = new PuppeteerBrowserAgent(browser, page, { }); ``` -Use `PuppeteerBrowserAgent.create(browser, options)` only when you do not already have a page. The factory uses `initialPage` when provided, otherwise it reuses the first existing browser page or creates a new page. +Use `new PuppeteerBrowserAgent(browser, page, options)` when you explicitly choose the initial active page. Use `PuppeteerBrowserAgent.create(browser, options)` when you want Midscene to choose or create the initial active page; the factory uses `initialPage` when provided, otherwise it reuses the first existing browser page or creates a new page. ### Connect Midscene Agent to a Remote Puppeteer Browser diff --git a/apps/site/docs/en/web-api-reference.mdx b/apps/site/docs/en/web-api-reference.mdx index c28c13ac82..88da996304 100644 --- a/apps/site/docs/en/web-api-reference.mdx +++ b/apps/site/docs/en/web-api-reference.mdx @@ -22,20 +22,22 @@ PuppeteerAgent, PlaywrightAgent, and Chrome Bridge share one action space; the M - `Reload` — Reload the page. - `GoBack` — Navigate back in history. -## PuppeteerAgent {#puppeteer-agent} +## PuppeteerPageAgent / PuppeteerAgent {#puppeteer-agent} Use Midscene against a Puppeteer-controlled browser when you need AI actions in your own Puppeteer workflows. +`PuppeteerPageAgent` is bound to one Puppeteer `Page`. `PuppeteerAgent` remains an alias for backward compatibility. + ### Import ```ts -import { PuppeteerAgent } from '@midscene/web/puppeteer'; +import { PuppeteerPageAgent } from '@midscene/web/puppeteer'; ``` ### Constructor ```ts -const agent = new PuppeteerAgent(page, { +const agent = new PuppeteerPageAgent(page, { // browser-specific options... }); ``` @@ -63,7 +65,7 @@ In addition to the base agent options, Puppeteer exposes: ### Browser agent -Use `PuppeteerBrowserAgent` when one Midscene Agent should manage page switching inside a Puppeteer browser. +Use `PuppeteerBrowserAgent` when one Midscene Agent should manage page switching inside a Puppeteer browser. It is bound to a browser instance, keeps one active page, and can optionally follow newly opened pages. ```ts const agent = new PuppeteerBrowserAgent(browser, page, { @@ -71,11 +73,14 @@ const agent = new PuppeteerBrowserAgent(browser, page, { }); ``` -- Constructor: `new PuppeteerBrowserAgent(browser, page, options?)` — Use this when you already have the initial page. -- Factory: `PuppeteerBrowserAgent.create(browser, options?)` — Use this when you do not already have a page. It uses `initialPage` if provided, otherwise the first existing browser page, or creates a new page. +- Constructor: `new PuppeteerBrowserAgent(browser, page, options?)` — Use this when you explicitly choose the initial active page. +- Factory: `PuppeteerBrowserAgent.create(browser, options?)` — Use this when you want Midscene to choose or create the initial active page. It uses `initialPage` if provided, otherwise the first existing browser page, or creates a new page. - `initialPage: Page` — Initial Puppeteer page for the factory. - `autoFollowNewPage: boolean` — Automatically switch the active page when the browser opens a new page. Default `false`. - `newPageTimeout: number` — Timeout for `waitForNewPage`. Default `5000`. +- `activePage: Page` — Current page controlled by the Browser Agent. +- `pages()` — List pages from the bound browser. +- `newPage()` — Create a new page and make it active. - `setActivePage(page: Page)` — Explicitly set which Puppeteer page the Browser Agent controls next. - `waitForNewPage(action?, options?)` — Wait for a newly opened page without implicitly switching the active page. @@ -129,20 +134,22 @@ await browser.disconnect(); - [Integrate with Puppeteer](./integrate-with-puppeteer) for installation, fixtures, and remote-CDP guidance. -## PlaywrightAgent {#playwright-agent} +## PlaywrightPageAgent / PlaywrightAgent {#playwright-agent} Use Midscene inside a Playwright browser for AI-driven testing or automation alongside your Playwright flows. +`PlaywrightPageAgent` is bound to one Playwright `Page`. `PlaywrightAgent` remains an alias for backward compatibility. + ### Import ```ts -import { PlaywrightAgent } from '@midscene/web/playwright'; +import { PlaywrightPageAgent } from '@midscene/web/playwright'; ``` ### Constructor ```ts -const agent = new PlaywrightAgent(page, { +const agent = new PlaywrightPageAgent(page, { // browser-specific options... }); ``` @@ -168,7 +175,7 @@ const agent = new PlaywrightAgent(page, { ### Browser agent -Use `PlaywrightBrowserAgent` when one Midscene Agent should manage page switching inside a Playwright browser context. +Use `PlaywrightBrowserAgent` when one Midscene Agent should manage page switching inside a Playwright browser context. It is bound to a browser context, keeps one active page, and can optionally follow newly opened pages. ```ts const agent = new PlaywrightBrowserAgent(context, page, { @@ -176,11 +183,14 @@ const agent = new PlaywrightBrowserAgent(context, page, { }); ``` -- Constructor: `new PlaywrightBrowserAgent(context, page, options?)` — Use this when you already have the initial page. -- Factory: `PlaywrightBrowserAgent.create(context, options?)` — Use this when you do not already have a page. It uses `initialPage` if provided, otherwise the first existing context page, or creates a new page. +- Constructor: `new PlaywrightBrowserAgent(context, page, options?)` — Use this when you explicitly choose the initial active page. +- Factory: `PlaywrightBrowserAgent.create(context, options?)` — Use this when you want Midscene to choose or create the initial active page. It uses `initialPage` if provided, otherwise the first existing context page, or creates a new page. - `initialPage: Page` — Initial Playwright page for the factory. - `autoFollowNewPage: boolean` — Automatically switch the active page when the context opens a new page. Default `false`. - `newPageTimeout: number` — Timeout for `waitForNewPage`. Default `5000`. +- `activePage: Page` — Current page controlled by the Browser Agent. +- `pages()` — List pages from the bound browser context. +- `newPage()` — Create a new page and make it active. - `setActivePage(page: Page)` — Explicitly set which Playwright page the Browser Agent controls next. - `waitForNewPage(action?, options?)` — Wait for a newly opened page without implicitly switching the active page. diff --git a/apps/site/docs/en/yaml-script-runner.mdx b/apps/site/docs/en/yaml-script-runner.mdx index 31103c50fb..04bfe51ebc 100644 --- a/apps/site/docs/en/yaml-script-runner.mdx +++ b/apps/site/docs/en/yaml-script-runner.mdx @@ -7,7 +7,7 @@ Midscene defines a YAML-based scripting format so you can quickly author automat For example, you can write a YAML script like this: ```yaml -web: +page: url: https://www.bing.com tasks: @@ -70,7 +70,7 @@ npm i @midscene/cli --save-dev Create `bing-search.yaml` to drive a web browser: ```yaml -web: +page: url: https://www.bing.com tasks: @@ -160,7 +160,7 @@ After execution, the output directory contains: ### Run in headed mode -> `web` scenarios only +> Web page scenarios only Headed mode opens the browser window. By default, scripts run headless. @@ -178,10 +178,10 @@ midscene /path/to/yaml --keep-window CDP mode lets YAML scripts connect to an existing browser instance via Chrome DevTools Protocol, without launching a new browser. This is useful for reusing an existing browser session, connecting to remote browsers, or cloud browser services. -Set `cdpEndpoint` in the `web` section: +Set `cdpEndpoint` in the `page` section: ```diff -web: +page: url: https://www.bing.com + cdpEndpoint: ws://localhost:9222/devtools/browser ``` @@ -194,12 +194,12 @@ CDP mode and bridge mode are mutually exclusive. In CDP mode, Midscene will only ### Use bridge mode -> `web` scenarios only +> Web page scenarios only Bridge mode lets YAML scripts drive your existing desktop browser so you can reuse cookies, extensions, or state. Install the Chrome extension, then add: ```diff -web: +page: url: https://www.bing.com + bridgeMode: newTabWithUrl ``` diff --git a/apps/site/docs/zh/automate-with-scripts-in-yaml.mdx b/apps/site/docs/zh/automate-with-scripts-in-yaml.mdx index f464d377dd..b03e702706 100644 --- a/apps/site/docs/zh/automate-with-scripts-in-yaml.mdx +++ b/apps/site/docs/zh/automate-with-scripts-in-yaml.mdx @@ -9,7 +9,7 @@ import SetupEnv from './common/setup-env.mdx'; 这里有一个示例，通过阅读它的内容，你应该已经理解了它的工作原理。 ```yaml -web: +page: url: https://www.bing.com tasks: @@ -41,10 +41,10 @@ tasks: 脚本文件使用 YAML 格式来描述自动化任务。它定义了要操作的目标（如网页或安卓应用）以及一系列要执行的步骤。 -一个标准的 `.yaml` 脚本文件包含 `web`、`android`、`ios`、`harmony` 或 `computer` 部分配置环境,可选的 `agent` 部分配置 AI Agent 行为,以及一个 `tasks` 部分来定义自动化任务。 +一个标准的 `.yaml` 脚本文件包含 `page`、`browser`、`web`、`android`、`ios`、`harmony` 或 `computer` 部分配置环境，可选的 `agent` 部分配置 AI Agent 行为，以及一个 `tasks` 部分来定义自动化任务。 ```yaml -web: +page: url: https://www.bing.com # tasks 部分定义了要执行的一系列步骤 @@ -56,6 +56,8 @@ tasks: - aiAssert: 结果显示天气信息 ``` +使用 `page:` 表示 page-level Agent。使用 `browser:` 表示一个 Agent 管理 browser 和它的 active page。`web:` 仍作为兼容入口保留；`web.mode: browser` 会映射到 BrowserAgent，普通 `web:` 会映射到 PageAgent。不要在同一份脚本里同时使用 `page`、`browser`、`web` 或已废弃的 `target`。 + ### `agent` 部分 `agent` 部分用于配置 AI Agent 的行为和测试报告相关选项。所有字段都是可选的。 @@ -140,10 +142,36 @@ tasks: ``` -### `web` 部分 +### Web target 部分 + +推荐的 page-level target： + +```yaml +page: + url: https://example.com +``` + +推荐的 browser-level target： + +```yaml +browser: + url: https://example.com + autoFollowNewPage: true +``` + +兼容写法： ```yaml web: + mode: browser + url: https://example.com + autoFollowNewPage: true +``` + +共享配置： + +```yaml +page: # 访问的 URL，必填。如果提供了 `serve` 参数，则提供相对路径 url: @@ -194,10 +222,11 @@ web: unstableLogContent: # 是否限制页面在当前 tab 打开，可选，默认 true + # 仅 page mode 可用，不要与 `browser:` 或 `web.mode: browser` 同时使用 forceSameTabNavigation: - # 是否使用 BrowserAgent 并自动继续在新打开的页面中执行，可选，默认 false - # 不能与 forceSameTabNavigation: true 同时使用 + # BrowserAgent 是否自动继续在新打开的页面中执行，可选，默认 false + # 仅 browser mode 可用。请使用 `browser:` 或 `web.mode: browser` autoFollowNewPage: # CDP 连接端点，可选。设置后通过 CDP 连接到已有浏览器实例，而非启动新浏览器。 diff --git a/apps/site/docs/zh/integrate-with-playwright.mdx b/apps/site/docs/zh/integrate-with-playwright.mdx index fdbf7095d1..592ba22537 100644 --- a/apps/site/docs/zh/integrate-with-playwright.mdx +++ b/apps/site/docs/zh/integrate-with-playwright.mdx @@ -264,11 +264,11 @@ npx playwright test ./e2e/ebay-search.spec.ts ### 关于在新标签页打开 -每个 Agent 实例都与对应的页面唯一绑定，为了方便开发者调试，Midscene 默认拦截了新 tab 的页面（如点击一个带有 `target="_blank"` 属性的链接），将其改为在当前页面打开。 +`PlaywrightAgent` 是 page-level Agent：每个实例都与对应的页面唯一绑定。为了方便开发者调试，Midscene 默认拦截了新 tab 的页面（如点击一个带有 `target="_blank"` 属性的链接），将其改为在当前页面打开。 如果你想恢复在新标签页打开的行为，同时让当前 Agent 仍留在原页面，可以设置 `forceSameTabNavigation` 为 `false`，并自行为每个新标签页创建新的 Agent 实例。 -如果后续操作要自动继续在新打开的标签页中执行，请使用 `PlaywrightBrowserAgent` 并开启 `autoFollowNewPage`。 +如果一个 Agent 需要管理整个 browser context 内的页面切换，请使用 `PlaywrightBrowserAgent`。如果后续操作要自动继续在新打开的标签页中执行，请开启 `autoFollowNewPage`。 ```typescript const mid = new PlaywrightBrowserAgent(context, page, { @@ -276,7 +276,7 @@ const mid = new PlaywrightBrowserAgent(context, page, { }); ``` -仅当你手上还没有现成的 page 时，才需要使用 `PlaywrightBrowserAgent.create(context, options)`。这个工厂会优先使用 `initialPage`，否则复用 context 里的第一个页面，或者创建一个新页面。 +当你要显式指定初始 active page 时，使用 `new PlaywrightBrowserAgent(context, page, options)`。当你希望 Midscene 自动选择或创建初始 active page 时，使用 `PlaywrightBrowserAgent.create(context, options)`；这个工厂会优先使用 `initialPage`，否则复用 context 里的第一个页面，或者创建一个新页面。 ### 连接远程 Playwright 浏览器并接入 Midscene Agent diff --git a/apps/site/docs/zh/integrate-with-puppeteer.mdx b/apps/site/docs/zh/integrate-with-puppeteer.mdx index 7e65dd4f37..777f527b76 100644 --- a/apps/site/docs/zh/integrate-with-puppeteer.mdx +++ b/apps/site/docs/zh/integrate-with-puppeteer.mdx @@ -104,11 +104,11 @@ npx tsx demo.ts ### 关于在新标签页打开 -每个 Agent 实例都与对应的页面唯一绑定，为了方便开发者调试，Midscene 默认拦截了新 tab 的页面（如点击一个带有 `target="_blank"` 属性的链接），将其改为在当前页面打开。 +`PuppeteerAgent` 是 page-level Agent：每个实例都与对应的页面唯一绑定。为了方便开发者调试，Midscene 默认拦截了新 tab 的页面（如点击一个带有 `target="_blank"` 属性的链接），将其改为在当前页面打开。 如果你想恢复在新标签页打开的行为，同时让当前 Agent 仍留在原页面，可以设置 `forceSameTabNavigation` 为 `false`，并自行为每个新标签页创建新的 Agent 实例。 -如果后续操作要自动继续在新打开的标签页中执行，请使用 `PuppeteerBrowserAgent` 并开启 `autoFollowNewPage`。 +如果一个 Agent 需要管理整个 browser 内的页面切换，请使用 `PuppeteerBrowserAgent`。如果后续操作要自动继续在新打开的标签页中执行，请开启 `autoFollowNewPage`。 ```typescript const mid = new PuppeteerBrowserAgent(browser, page, { @@ -116,7 +116,7 @@ const mid = new PuppeteerBrowserAgent(browser, page, { }); ``` -仅当你手上还没有现成的 page 时，才需要使用 `PuppeteerBrowserAgent.create(browser, options)`。这个工厂会优先使用 `initialPage`，否则复用浏览器里的第一个页面，或者创建一个新页面。 +当你要显式指定初始 active page 时，使用 `new PuppeteerBrowserAgent(browser, page, options)`。当你希望 Midscene 自动选择或创建初始 active page 时，使用 `PuppeteerBrowserAgent.create(browser, options)`；这个工厂会优先使用 `initialPage`，否则复用浏览器里的第一个页面，或者创建一个新页面。 ### 连接远程 Puppeteer 浏览器并接入 Midscene Agent diff --git a/apps/site/docs/zh/web-api-reference.mdx b/apps/site/docs/zh/web-api-reference.mdx index ce4e96adf2..3eeebfbd50 100644 --- a/apps/site/docs/zh/web-api-reference.mdx +++ b/apps/site/docs/zh/web-api-reference.mdx @@ -22,20 +22,22 @@ PuppeteerAgent、PlaywrightAgent 和 Chrome Bridge 共用一套 Action Space，M - `Reload` —— 刷新当前页面。 - `GoBack` —— 浏览器后退。 -## PuppeteerAgent {#puppeteer-agent} +## PuppeteerPageAgent / PuppeteerAgent {#puppeteer-agent} 当你需要在 Puppeteer 控制的浏览器里复用 Midscene 的 AI 操作能力时使用。 +`PuppeteerPageAgent` 绑定单个 Puppeteer `Page`。`PuppeteerAgent` 仍作为兼容别名保留。 + ### 导入 ```ts -import { PuppeteerAgent } from '@midscene/web/puppeteer'; +import { PuppeteerPageAgent } from '@midscene/web/puppeteer'; ``` ### 构造器 ```ts -const agent = new PuppeteerAgent(page, { +const agent = new PuppeteerPageAgent(page, { // 浏览器特有配置... }); ``` @@ -63,7 +65,7 @@ const agent = new PuppeteerAgent(page, { ### Browser Agent -当一个 Midscene Agent 需要管理 Puppeteer 浏览器内的页面切换时，使用 `PuppeteerBrowserAgent`。 +当一个 Midscene Agent 需要管理 Puppeteer 浏览器内的页面切换时，使用 `PuppeteerBrowserAgent`。它绑定 browser 实例，维护一个 active page，并且可以选择自动跟随新打开的页面。 ```ts const agent = new PuppeteerBrowserAgent(browser, page, { @@ -71,11 +73,14 @@ const agent = new PuppeteerBrowserAgent(browser, page, { }); ``` -- 构造函数：`new PuppeteerBrowserAgent(browser, page, options?)` —— 已经有初始页面时使用。 -- 工厂方法：`PuppeteerBrowserAgent.create(browser, options?)` —— 手上还没有 page 时使用。它会优先使用 `initialPage`，否则复用浏览器里的第一个页面，或者创建一个新页面。 +- 构造函数：`new PuppeteerBrowserAgent(browser, page, options?)` —— 你要显式指定初始 active page 时使用。 +- 工厂方法：`PuppeteerBrowserAgent.create(browser, options?)` —— 你希望 Midscene 自动选择或创建初始 active page 时使用。它会优先使用 `initialPage`，否则复用浏览器里的第一个页面，或者创建一个新页面。 - `initialPage: Page` —— 工厂方法使用的初始 Puppeteer 页面。 - `autoFollowNewPage: boolean` —— 浏览器打开新页面时是否自动切换 active page，默认 `false`。 - `newPageTimeout: number` —— `waitForNewPage` 的超时时间，默认 `5000`。 +- `activePage: Page` —— Browser Agent 当前控制的页面。 +- `pages()` —— 列出绑定 browser 中的页面。 +- `newPage()` —— 创建新页面并将其设为 active page。 - `setActivePage(page: Page)` —— 显式指定 Browser Agent 接下来控制哪个 Puppeteer 页面。 - `waitForNewPage(action?, options?)` —— 等待新打开的页面，但不会隐式切换 active page。 @@ -129,20 +134,22 @@ await browser.disconnect(); - [集成到 Puppeteer](./integrate-with-puppeteer) 获取安装、Fixture 与远程 CDP 配置。 -## PlaywrightAgent {#playwright-agent} +## PlaywrightPageAgent / PlaywrightAgent {#playwright-agent} 在 Playwright 浏览器中使用 Midscene 以支持带 AI 的测试或自动化流程。 +`PlaywrightPageAgent` 绑定单个 Playwright `Page`。`PlaywrightAgent` 仍作为兼容别名保留。 + ### 导入 ```ts -import { PlaywrightAgent } from '@midscene/web/playwright'; +import { PlaywrightPageAgent } from '@midscene/web/playwright'; ``` ### 构造器 ```ts -const agent = new PlaywrightAgent(page, { +const agent = new PlaywrightPageAgent(page, { // 浏览器特有配置... }); ``` @@ -168,7 +175,7 @@ const agent = new PlaywrightAgent(page, { ### Browser Agent -当一个 Midscene Agent 需要管理 Playwright browser context 内的页面切换时，使用 `PlaywrightBrowserAgent`。 +当一个 Midscene Agent 需要管理 Playwright browser context 内的页面切换时，使用 `PlaywrightBrowserAgent`。它绑定 browser context，维护一个 active page，并且可以选择自动跟随新打开的页面。 ```ts const agent = new PlaywrightBrowserAgent(context, page, { @@ -176,11 +183,14 @@ const agent = new PlaywrightBrowserAgent(context, page, { }); ``` -- 构造函数：`new PlaywrightBrowserAgent(context, page, options?)` —— 已经有初始页面时使用。 -- 工厂方法：`PlaywrightBrowserAgent.create(context, options?)` —— 手上还没有 page 时使用。它会优先使用 `initialPage`，否则复用 context 里的第一个页面，或者创建一个新页面。 +- 构造函数：`new PlaywrightBrowserAgent(context, page, options?)` —— 你要显式指定初始 active page 时使用。 +- 工厂方法：`PlaywrightBrowserAgent.create(context, options?)` —— 你希望 Midscene 自动选择或创建初始 active page 时使用。它会优先使用 `initialPage`，否则复用 context 里的第一个页面，或者创建一个新页面。 - `initialPage: Page` —— 工厂方法使用的初始 Playwright 页面。 - `autoFollowNewPage: boolean` —— context 打开新页面时是否自动切换 active page，默认 `false`。 - `newPageTimeout: number` —— `waitForNewPage` 的超时时间，默认 `5000`。 +- `activePage: Page` —— Browser Agent 当前控制的页面。 +- `pages()` —— 列出绑定 browser context 中的页面。 +- `newPage()` —— 创建新页面并将其设为 active page。 - `setActivePage(page: Page)` —— 显式指定 Browser Agent 接下来控制哪个 Playwright 页面。 - `waitForNewPage(action?, options?)` —— 等待新打开的页面，但不会隐式切换 active page。 diff --git a/apps/site/docs/zh/yaml-script-runner.mdx b/apps/site/docs/zh/yaml-script-runner.mdx index 0128eeb7f7..3cd4cb82c0 100644 --- a/apps/site/docs/zh/yaml-script-runner.mdx +++ b/apps/site/docs/zh/yaml-script-runner.mdx @@ -7,7 +7,7 @@ Midscene 定义了一种 YAML 格式的脚本，方便开发者快速编写自 举例来说，你可以编写如下 YAML 格式脚本示例： ```yaml -web: +page: url: https://www.bing.com tasks: @@ -73,7 +73,7 @@ npm i @midscene/cli --save-dev 编写一个名为 `bing-search.yaml` 的文件来驱动 Web 浏览器： ```yaml -web: +page: url: https://www.bing.com tasks: @@ -164,7 +164,7 @@ midscene './scripts/**/*.yaml' ### 运行在可视化（Headed）模式 -> 仅适用于 `web` 场景 +> 仅适用于 Web page 场景 Headed 模式会打开浏览器窗口。默认情况下脚本在无头模式运行。 @@ -182,10 +182,10 @@ midscene /path/to/yaml --keep-window CDP 模式可以让 YAML 脚本通过 Chrome DevTools Protocol 连接到已有的浏览器实例，无需启动新浏览器。适用于需要复用已有浏览器会话、连接远程浏览器或云端浏览器服务的场景。 -在 `web` 配置中设置 `cdpEndpoint`： +在 `page` 配置中设置 `cdpEndpoint`： ```diff -web: +page: url: https://www.bing.com + cdpEndpoint: ws://localhost:9222/devtools/browser ``` @@ -198,12 +198,12 @@ CDP 模式与桥接模式互斥，不可同时使用。CDP 模式下 Midscene ### 使用桥接模式 -> 仅适用于 `web` 场景 +> 仅适用于 Web page 场景 -使用桥接模式可以让 YAML 脚本驱动现有的桌面浏览器，便于复用 Cookies、插件或已有状态。先安装 Chrome 扩展，然后在 `web` 配置中加入： +使用桥接模式可以让 YAML 脚本驱动现有的桌面浏览器，便于复用 Cookies、插件或已有状态。先安装 Chrome 扩展，然后在 `page` 配置中加入： ```diff -web: +page: url: https://www.bing.com + bridgeMode: newTabWithUrl ``` diff --git a/packages/cli/src/config-factory.ts b/packages/cli/src/config-factory.ts index d851756ab6..444a6c4d2a 100644 --- a/packages/cli/src/config-factory.ts +++ b/packages/cli/src/config-factory.ts @@ -7,7 +7,7 @@ import type { MidsceneYamlScriptIOSEnv, MidsceneYamlScriptWebEnv, } from '@midscene/core'; -import { interpolateEnvVars } from '@midscene/core/yaml'; +import { interpolateEnvVars, resolveWebTarget } from '@midscene/core/yaml'; import { load as yamlLoad } from 'js-yaml'; import merge from 'lodash.merge'; import type { BatchRunnerConfig } from './batch-runner'; @@ -34,6 +34,9 @@ export interface ConfigFactoryOptions { keepWindow?: boolean; dotenvOverride?: boolean; dotenvDebug?: boolean; + target?: Partial; + page?: Partial; + browser?: Partial; web?: Partial; android?: Partial; ios?: Partial; @@ -46,6 +49,8 @@ export interface ParsedConfig { retry: number; summary: string; shareBrowserContext: boolean; + page?: MidsceneYamlScriptWebEnv; + browser?: MidsceneYamlScriptWebEnv; web?: MidsceneYamlScriptWebEnv; android?: MidsceneYamlScriptAndroidEnv; ios?: MidsceneYamlScriptIOSEnv; @@ -100,6 +105,8 @@ export async function parseConfigYaml( throw new Error('Config YAML must contain a "files" array'); } + resolveWebTarget(configYaml); + // Expand file patterns using glob const files = await expandFilePatterns(configYaml?.files, basePath); @@ -122,7 +129,10 @@ export async function parseConfigYaml( summary: configYaml.summary ?? defaultSummary, shareBrowserContext: configYaml.shareBrowserContext ?? defaultConfig.shareBrowserContext, + page: configYaml.page, + browser: configYaml.browser, web: configYaml.web, + target: configYaml.target, android: configYaml.android, ios: configYaml.ios, patterns: configYaml.files, @@ -143,15 +153,20 @@ export async function createConfig( const parsedConfig = await parseConfigYaml(configYamlPath); const globalConfig = merge( { + page: parsedConfig.page, + browser: parsedConfig.browser, web: parsedConfig.web, android: parsedConfig.android, ios: parsedConfig.ios, target: parsedConfig.target, }, { + page: options?.page, + browser: options?.browser, web: options?.web, android: options?.android, ios: options?.ios, + target: options?.target, }, ); @@ -213,7 +228,10 @@ export async function createFilesConfig( dotenvOverride: options.dotenvOverride ?? defaultConfig.dotenvOverride, dotenvDebug: options.dotenvDebug ?? defaultConfig.dotenvDebug, globalConfig: { + page: options.page as MidsceneYamlScriptWebEnv | undefined, + browser: options.browser as MidsceneYamlScriptWebEnv | undefined, web: options.web as MidsceneYamlScriptWebEnv | undefined, + target: options.target as MidsceneYamlScriptWebEnv | undefined, android: options.android as MidsceneYamlScriptAndroidEnv | undefined, ios: options.ios as MidsceneYamlScriptIOSEnv | undefined, }, diff --git a/packages/cli/src/create-yaml-player.ts b/packages/cli/src/create-yaml-player.ts index d96f618f02..8541b5e9a6 100644 --- a/packages/cli/src/create-yaml-player.ts +++ b/packages/cli/src/create-yaml-player.ts @@ -1,6 +1,10 @@ import { readFileSync } from 'node:fs'; import path, { basename, extname, join } from 'node:path'; -import { ScriptPlayer, parseYamlScript } from '@midscene/core/yaml'; +import { + ScriptPlayer, + parseYamlScript, + resolveWebTarget, +} from '@midscene/core/yaml'; import { createServer } from 'http-server'; import assert from 'node:assert'; @@ -115,11 +119,14 @@ export async function createYamlPlayer( clonedYamlScript, async () => { const freeFn: FreeFn[] = []; - const webTarget = clonedYamlScript.web || clonedYamlScript.target; + const resolvedWebTarget = resolveWebTarget(clonedYamlScript); + const webTarget = resolvedWebTarget?.target as + | MidsceneYamlScriptWebEnv + | undefined; // Validate that only one target type is specified const targetCount = [ - typeof webTarget !== 'undefined', + typeof resolvedWebTarget !== 'undefined', typeof clonedYamlScript.android !== 'undefined', typeof clonedYamlScript.ios !== 'undefined', typeof clonedYamlScript.harmony !== 'undefined', @@ -129,7 +136,7 @@ export async function createYamlPlayer( if (targetCount > 1) { const specifiedTargets = [ - typeof webTarget !== 'undefined' ? 'web' : null, + resolvedWebTarget?.source ?? null, typeof clonedYamlScript.android !== 'undefined' ? 'android' : null, typeof clonedYamlScript.ios !== 'undefined' ? 'ios' : null, typeof clonedYamlScript.harmony !== 'undefined' ? 'harmony' : null, @@ -140,15 +147,15 @@ export async function createYamlPlayer( ].filter(Boolean); throw new Error( - `Only one target type can be specified, but found multiple: ${specifiedTargets.join(', ')}. Please specify only one of: web, android, ios, harmony, computer, or interface.`, + `Only one target type can be specified, but found multiple: ${specifiedTargets.join(', ')}. Please specify only one of: page, browser, web, android, ios, harmony, computer, or interface.`, ); } // handle new web config if (typeof webTarget !== 'undefined') { - if (typeof clonedYamlScript.target !== 'undefined') { + if (resolvedWebTarget?.source === 'target') { console.warn( - 'target is deprecated, please use web instead. See https://midscenejs.com/automate-with-scripts-in-yaml for more information. Sorry for the inconvenience.', + 'target is deprecated, please use page or browser instead. See https://midscenejs.com/automate-with-scripts-in-yaml for more information. Sorry for the inconvenience.', ); } @@ -181,6 +188,12 @@ export async function createYamlPlayer( ); } + if (webTarget.mode === 'browser' && webTarget.bridgeMode) { + throw new Error( + '[midscene] browser mode does not support bridgeMode. Use page: or web.mode: page for bridge mode.', + ); + } + // CDP mode: connect to an existing browser via Chrome DevTools Protocol if (webTarget.cdpEndpoint) { // Use the shared browser from batch-runner if available (shareBrowserContext), diff --git a/packages/cli/src/framework/yaml-case.ts b/packages/cli/src/framework/yaml-case.ts index cc4234e259..85d191d6c9 100644 --- a/packages/cli/src/framework/yaml-case.ts +++ b/packages/cli/src/framework/yaml-case.ts @@ -14,6 +14,8 @@ import merge from 'lodash.merge'; import { createYamlPlayer } from '../create-yaml-player'; export interface RunYamlCaseGlobalConfig { + page?: Partial; + browser?: Partial; web?: Partial; android?: Partial; ios?: Partial; diff --git a/packages/cli/src/yaml-batch-executor.ts b/packages/cli/src/yaml-batch-executor.ts index 528d0f00c5..ef93fb3980 100644 --- a/packages/cli/src/yaml-batch-executor.ts +++ b/packages/cli/src/yaml-batch-executor.ts @@ -7,7 +7,11 @@ import type { MidsceneYamlScriptIOSEnv, MidsceneYamlScriptWebEnv, } from '@midscene/core'; -import { type ScriptPlayer, parseYamlScript } from '@midscene/core/yaml'; +import { + type ScriptPlayer, + parseYamlScript, + resolveWebTarget, +} from '@midscene/core/yaml'; import { buildChromeArgs, buildDownloadBehavior, @@ -48,6 +52,8 @@ export interface BatchRunnerConfig { summary: string; shareBrowserContext: boolean; globalConfig?: { + page?: Partial; + browser?: Partial; web?: Partial; android?: Partial; ios?: Partial; @@ -114,14 +120,13 @@ class YamlBatchExecutor { // Now, check if any of the tasks require a web browser const needsBrowser = fileContextList.some( - (ctx) => - Object.keys( - ctx.executionConfig.web || ctx.executionConfig.target || {}, - ).length > 0, + (ctx) => typeof resolveWebTarget(ctx.executionConfig) !== 'undefined', ); if (needsBrowser && this.config.shareBrowserContext) { - const globalWebConfig = this.config.globalConfig?.web; + const globalWebConfig = resolveWebTarget( + this.config.globalConfig ?? {}, + )?.target; if (globalWebConfig?.cdpEndpoint) { // CDP mode: connect to an existing browser @@ -180,7 +185,8 @@ class YamlBatchExecutor { } finally { if (browser && !this.config.keepWindow) { // For CDP mode, disconnect instead of closing the externally managed browser - const isCdp = !!this.config.globalConfig?.web?.cdpEndpoint; + const isCdp = !!resolveWebTarget(this.config.globalConfig ?? {})?.target + .cdpEndpoint; if (isCdp) { browser.disconnect(); } else { @@ -205,21 +211,6 @@ class YamlBatchExecutor { // Deep clone to avoid mutation const clonedFileConfig = JSON.parse(JSON.stringify(fileConfig)); - // Normalize deprecated 'target' to 'web' - if (clonedFileConfig.target) { - clonedFileConfig.web = { - ...clonedFileConfig.target, - ...clonedFileConfig.web, - }; - // biome-ignore lint/performance/noDelete: - delete clonedFileConfig.target; - } - if (globalConfig?.target) { - globalConfig.web = { ...globalConfig.target, ...globalConfig.web }; - // biome-ignore lint/performance/noDelete: - delete globalConfig.target; - } - // Start with the file's config, then merge the global config from the index file, // which has already been merged with command-line options. const executionConfig = merge(clonedFileConfig, globalConfig); diff --git a/packages/cli/tests/unit-test/config-factory.test.ts b/packages/cli/tests/unit-test/config-factory.test.ts index 848d77cec4..ffd3388829 100644 --- a/packages/cli/tests/unit-test/config-factory.test.ts +++ b/packages/cli/tests/unit-test/config-factory.test.ts @@ -18,6 +18,37 @@ vi.mock('@/cli-utils', () => ({ vi.mock('@midscene/core/yaml', () => ({ interpolateEnvVars: vi.fn((content) => content), + resolveWebTarget: vi.fn((config) => { + const sources = ['page', 'browser', 'web', 'target'] as const; + const entries = sources + .map((source) => [source, config[source]] as const) + .filter(([, value]) => typeof value !== 'undefined'); + + if (entries.length === 0) { + return undefined; + } + + if (entries.length > 1) { + throw new Error('Only one web target can be specified'); + } + + const [source, target] = entries[0]; + const mode = + source === 'page' + ? 'page' + : source === 'browser' + ? 'browser' + : (target.mode ?? 'page'); + + return { + source, + mode, + target: { + ...target, + mode, + }, + }; + }), })); vi.mock('js-yaml', () => ({ @@ -62,7 +93,10 @@ summary: "yaml-summary.json" keepWindow: true, dotenvOverride: true, dotenvDebug: false, + page: undefined, + browser: undefined, web: { url: 'http://example.com', userAgent: 'yaml-ua' }, + target: undefined, android: { deviceId: 'yaml-device' }, summary: 'yaml-summary.json', }; @@ -257,12 +291,20 @@ concurrent: 2 const expectedGlobalConfig = merge( { + page: mockParsedYaml.page, + browser: mockParsedYaml.browser, web: mockParsedYaml.web, android: mockParsedYaml.android, + ios: mockParsedYaml.ios, + target: mockParsedYaml.target, }, { + page: cmdLineOptions.page, + browser: cmdLineOptions.browser, web: cmdLineOptions.web, android: cmdLineOptions.android, + ios: cmdLineOptions.ios, + target: cmdLineOptions.target, }, ); @@ -373,7 +415,10 @@ concurrent: 2 dotenvOverride: false, dotenvDebug: false, globalConfig: { + page: undefined, + browser: undefined, web: undefined, + target: undefined, android: undefined, ios: undefined, }, @@ -426,8 +471,12 @@ concurrent: 2 dotenvOverride: true, dotenvDebug: false, globalConfig: { + page: undefined, + browser: undefined, web: { userAgent: 'custom-ua' }, + target: undefined, android: { deviceId: 'custom-device' }, + ios: undefined, }, }); expect(matchYamlFiles).toHaveBeenCalledWith(patterns[0], { @@ -479,11 +528,14 @@ concurrent: 2 dotenvOverride: false, dotenvDebug: false, globalConfig: { + page: undefined, + browser: undefined, web: { userAgent: 'Doc Agent', viewportWidth: 1440, viewportHeight: 900, }, + target: undefined, android: { deviceId: 'android-doc-device', }, diff --git a/packages/cli/tests/unit-test/create-yaml-player.test.ts b/packages/cli/tests/unit-test/create-yaml-player.test.ts index bd017c029f..208159f65a 100644 --- a/packages/cli/tests/unit-test/create-yaml-player.test.ts +++ b/packages/cli/tests/unit-test/create-yaml-player.test.ts @@ -171,6 +171,105 @@ describe('create-yaml-player', () => { expect(result).toBe(mockPlayer); }); + test('should pass explicit page target to puppeteer launcher', async () => { + const mockScript: MidsceneYamlScript = { + page: { + url: 'http://example.com', + }, + tasks: [], + }; + const mockAgent = { destroy: vi.fn() }; + let setupFnCallback: (() => Promise) | undefined; + + vi.mocked(puppeteerAgentForTarget).mockResolvedValue({ + agent: mockAgent as any, + freeFn: [], + }); + vi.mocked(ScriptPlayer).mockImplementation((script, setupFn) => { + setupFnCallback = setupFn as () => Promise; + return { + addCleanup: vi.fn(), + } as unknown as ScriptPlayer; + }); + + await createYamlPlayer(mockFilePath, mockScript); + await setupFnCallback?.(); + + expect(puppeteerAgentForTarget).toHaveBeenCalledWith( + expect.objectContaining({ + mode: 'page', + url: 'http://example.com', + }), + expect.any(Object), + undefined, + undefined, + ); + }); + + test('should pass explicit browser target to puppeteer launcher', async () => { + const mockScript: MidsceneYamlScript = { + browser: { + url: 'http://example.com', + autoFollowNewPage: true, + }, + tasks: [], + }; + const mockAgent = { destroy: vi.fn() }; + let setupFnCallback: (() => Promise) | undefined; + + vi.mocked(puppeteerAgentForTarget).mockResolvedValue({ + agent: mockAgent as any, + freeFn: [], + }); + vi.mocked(ScriptPlayer).mockImplementation((script, setupFn) => { + setupFnCallback = setupFn as () => Promise; + return { + addCleanup: vi.fn(), + } as unknown as ScriptPlayer; + }); + + await createYamlPlayer(mockFilePath, mockScript); + await setupFnCallback?.(); + + expect(puppeteerAgentForTarget).toHaveBeenCalledWith( + expect.objectContaining({ + mode: 'browser', + url: 'http://example.com', + autoFollowNewPage: true, + }), + expect.any(Object), + undefined, + undefined, + ); + }); + + test('should reject conflicting web targets during setup', async () => { + const mockScript: MidsceneYamlScript = { + page: { + url: 'http://example.com/page', + }, + browser: { + url: 'http://example.com/browser', + }, + tasks: [], + }; + let setupFnCallback: (() => Promise) | undefined; + + vi.mocked(ScriptPlayer).mockImplementation((script, setupFn) => { + setupFnCallback = setupFn as () => Promise; + return { + addCleanup: vi.fn(), + } as unknown as ScriptPlayer; + }); + + await createYamlPlayer(mockFilePath, mockScript); + + expect(setupFnCallback).toBeDefined(); + await expect(setupFnCallback!()).rejects.toThrow( + 'Only one web target can be specified', + ); + }); + test('should create player with bridge mode configuration', async () => { const mockScript: MidsceneYamlScript = { web: { diff --git a/packages/core/src/yaml.ts b/packages/core/src/yaml.ts index dc4795ecaf..5d40ccff5f 100644 --- a/packages/core/src/yaml.ts +++ b/packages/core/src/yaml.ts @@ -55,6 +55,8 @@ export interface MidsceneYamlScript { // @deprecated target?: MidsceneYamlScriptWebEnv; + page?: MidsceneYamlScriptWebEnv; + browser?: MidsceneYamlScriptWebEnv; web?: MidsceneYamlScriptWebEnv; android?: MidsceneYamlScriptAndroidEnv; ios?: MidsceneYamlScriptIOSEnv; @@ -128,6 +130,8 @@ export interface MidsceneYamlScriptEnvGeneralInterface { export interface MidsceneYamlScriptWebEnv extends MidsceneYamlScriptConfig, MidsceneYamlScriptAgentOpt { + mode?: 'page' | 'browser'; + // for web only serve?: string; url: string; @@ -333,6 +337,10 @@ export interface MidsceneYamlConfig { retry?: number; summary?: string; shareBrowserContext?: boolean; + /** @deprecated Use `web`, `page`, or `browser` instead. */ + target?: MidsceneYamlScriptWebEnv; + page?: MidsceneYamlScriptWebEnv; + browser?: MidsceneYamlScriptWebEnv; web?: MidsceneYamlScriptWebEnv; android?: MidsceneYamlScriptAndroidEnv; ios?: MidsceneYamlScriptIOSEnv; diff --git a/packages/core/src/yaml/builder.ts b/packages/core/src/yaml/builder.ts index 8ad605ad81..dd2bc28799 100644 --- a/packages/core/src/yaml/builder.ts +++ b/packages/core/src/yaml/builder.ts @@ -10,7 +10,7 @@ export function buildYaml( tasks: MidsceneYamlTask[], ) { const result: MidsceneYamlScript = { - target: env, + page: env, tasks, }; diff --git a/packages/core/src/yaml/utils.ts b/packages/core/src/yaml/utils.ts index 6b5b8fb806..eaffb6344a 100644 --- a/packages/core/src/yaml/utils.ts +++ b/packages/core/src/yaml/utils.ts @@ -3,6 +3,7 @@ import type { DetailedLocateParam, LocateOption, MidsceneYamlScript, + MidsceneYamlScriptWebEnv, } from '@/types'; import { getDebug } from '@midscene/shared/logger'; import { assert } from '@midscene/shared/utils'; @@ -13,6 +14,108 @@ const debugUtils = getDebug('yaml:utils'); const topLevelTasksPattern = /^tasks\s*:/; const topLevelYamlKeyPattern = /^[^\s#][^:]*:/; +export type WebTargetSource = 'page' | 'browser' | 'web' | 'target'; + +export type ResolvedWebTarget = { + source: WebTargetSource; + target: Partial & { mode: 'page' | 'browser' }; + mode: 'page' | 'browser'; +}; + +export type WebTargetConfig = Partial< + Record> +>; + +const webTargetSources: WebTargetSource[] = [ + 'page', + 'browser', + 'web', + 'target', +]; + +export function resolveWebTarget( + config: WebTargetConfig, +): ResolvedWebTarget | undefined { + const entries = webTargetSources + .map((source) => [source, config[source]] as const) + .filter( + ( + entry, + ): entry is readonly [ + WebTargetSource, + Partial, + ] => typeof entry[1] !== 'undefined', + ); + + if (entries.length === 0) { + return undefined; + } + + if (entries.length > 1) { + const specifiedTargets = entries.map(([source]) => source); + throw new Error( + `[midscene] Only one web target can be specified, but found multiple: ${specifiedTargets.join( + ', ', + )}. Please specify only one of: page, browser, web, or target.`, + ); + } + + const [source, target] = entries[0]; + const explicitMode = target.mode; + if ( + typeof explicitMode !== 'undefined' && + explicitMode !== 'page' && + explicitMode !== 'browser' + ) { + throw new Error( + `[midscene] web target mode must be either "page" or "browser", but got "${explicitMode}".`, + ); + } + + if (source === 'page' && explicitMode === 'browser') { + throw new Error( + '[midscene] page target cannot use mode: browser. Use browser: instead.', + ); + } + + if (source === 'browser' && explicitMode === 'page') { + throw new Error( + '[midscene] browser target cannot use mode: page. Use page: instead.', + ); + } + + const mode = + source === 'page' + ? 'page' + : source === 'browser' + ? 'browser' + : (explicitMode ?? 'page'); + + if (mode === 'page' && target.autoFollowNewPage) { + throw new Error( + '[midscene] autoFollowNewPage requires browser mode. Use browser: or web.mode: browser.', + ); + } + + if ( + mode === 'browser' && + typeof target.forceSameTabNavigation !== 'undefined' + ) { + throw new Error( + '[midscene] forceSameTabNavigation cannot be used in browser mode. Use page: or web.mode: page when same-tab navigation is required.', + ); + } + + return { + source, + mode, + target: { + ...target, + mode, + }, + }; +} + function interpolateEnvVarRefs( value: string, keepUnresolvedRefs = false, @@ -152,6 +255,7 @@ export function parseYamlScript( }) as MidsceneYamlScript; const pathTip = filePath ? `, failed to load ${filePath}` : ''; + resolveWebTarget(obj); assert(obj.tasks, `property "tasks" is required in yaml script ${pathTip}`); assert( Array.isArray(obj.tasks), diff --git a/packages/web-integration/src/common/browser-agent.ts b/packages/web-integration/src/common/browser-agent.ts new file mode 100644 index 0000000000..2de246ab4c --- /dev/null +++ b/packages/web-integration/src/common/browser-agent.ts @@ -0,0 +1,167 @@ +import type { DebugFunction } from '@midscene/shared/logger'; + +export type BrowserAgentAdapter = { + pages(): Page[] | Promise; + newPage(): Promise; + isPageClosed(page: Page): boolean; + bringToFront(page: Page): Promise | void; + onNewPage(handler: (event: NewPageEvent) => void): void; + offNewPage(handler: (event: NewPageEvent) => void): void; + resolveNewPage(event: NewPageEvent): Page | Promise | null; + isNewPageEvent?: (event: NewPageEvent) => boolean; +}; + +export type BrowserAgentPageControllerOptions = { + agentName: string; + adapter: BrowserAgentAdapter; + getActivePage(): Page; + setActivePageValue(page: Page): void; + autoFollowNewPage: boolean; + newPageTimeout: number; + debug: DebugFunction; +}; + +export class BrowserAgentPageController { + private readonly agentName: string; + private readonly adapter: BrowserAgentAdapter; + private readonly getActivePageValue: () => Page; + private readonly setActivePageValue: (page: Page) => void; + private readonly newPageTimeout: number; + private readonly debug: DebugFunction; + + private readonly newPageHandler = (event: NewPageEvent) => { + void this.followNewPage(event); + }; + + constructor(options: BrowserAgentPageControllerOptions) { + this.agentName = options.agentName; + this.adapter = options.adapter; + this.getActivePageValue = options.getActivePage; + this.setActivePageValue = options.setActivePageValue; + this.newPageTimeout = options.newPageTimeout; + this.debug = options.debug; + + if (options.autoFollowNewPage) { + this.adapter.onNewPage(this.newPageHandler); + } + } + + get activePage() { + return this.getActivePageValue(); + } + + pages() { + return this.adapter.pages(); + } + + async newPage() { + const page = await this.adapter.newPage(); + await this.setActivePage(page); + return page; + } + + async setActivePage(page: Page) { + if (!page || this.adapter.isPageClosed(page)) { + throw new Error( + `[midscene] Cannot set ${this.agentName} active page to a closed or invalid page.`, + ); + } + + this.setActivePageValue(page); + try { + await this.adapter.bringToFront(page); + } catch (error) { + this.debug(`failed to bring page to front: ${error}`); + } + } + + async waitForNewPage( + action?: () => Promise | unknown, + opts?: { timeout?: number }, + ) { + const waiter = this.createNewPageWaiter(opts?.timeout); + + try { + await action?.(); + return await waiter.promise; + } catch (error) { + waiter.dispose(); + throw error; + } + } + + destroy() { + this.adapter.offNewPage(this.newPageHandler); + } + + private async followNewPage(event: NewPageEvent) { + if (!this.isNewPageEvent(event)) { + return; + } + + try { + const page = await this.adapter.resolveNewPage(event); + if (page) { + await this.setActivePage(page); + } + } catch (error) { + this.debug(`failed to follow new page: ${error}`); + } + } + + private isNewPageEvent(event: NewPageEvent) { + return this.adapter.isNewPageEvent?.(event) ?? true; + } + + private createNewPageWaiter(timeout = this.newPageTimeout) { + let settled = false; + + const dispose = () => { + this.adapter.offNewPage(handler); + clearTimeout(timer); + }; + + const handler = async (event: NewPageEvent) => { + if (settled || !this.isNewPageEvent(event)) { + return; + } + + settled = true; + dispose(); + + try { + const page = await this.adapter.resolveNewPage(event); + if (!page) { + throw new Error('new target did not resolve to a page'); + } + resolvePage(page); + } catch (error) { + rejectPage(error); + } + }; + + let resolvePage!: (page: Page) => void; + let rejectPage!: (error: unknown) => void; + const promise = new Promise((resolve, reject) => { + resolvePage = resolve; + rejectPage = reject; + }); + + const timer = setTimeout(() => { + if (settled) { + return; + } + settled = true; + dispose(); + rejectPage( + new Error( + `[midscene] Timed out waiting for a new ${this.agentName} page after ${timeout}ms.`, + ), + ); + }, timeout); + + this.adapter.onNewPage(handler); + + return { promise, dispose }; + } +} diff --git a/packages/web-integration/src/index.ts b/packages/web-integration/src/index.ts index c571378ba3..0fe04a22b3 100644 --- a/packages/web-integration/src/index.ts +++ b/packages/web-integration/src/index.ts @@ -4,12 +4,14 @@ export type { PlayWrightAiFixtureType } from './playwright'; export { Agent as PageAgent, type AgentOpt } from '@midscene/core/agent'; export { PuppeteerAgent, + PuppeteerPageAgent, PuppeteerBrowserAgent, type PuppeteerBrowserAgentCreateOpt, type PuppeteerBrowserAgentOpt, } from './puppeteer'; export { PlaywrightAgent, + PlaywrightPageAgent, PlaywrightBrowserAgent, type PlaywrightBrowserAgentCreateOpt, type PlaywrightBrowserAgentOpt, diff --git a/packages/web-integration/src/playwright/agent.ts b/packages/web-integration/src/playwright/agent.ts index dbb6bb2d72..83ecf289b8 100644 --- a/packages/web-integration/src/playwright/agent.ts +++ b/packages/web-integration/src/playwright/agent.ts @@ -1,41 +1,4 @@ -import { - applyForceChromeSelectRendering, - isRetryableBrowserNavigationError, -} from '@/common/web-agent'; -import type { WebPageAgentOpt } from '@/web-element'; -import { Agent as PageAgent } from '@midscene/core/agent'; -import { getDebug } from '@midscene/shared/logger'; -import type { Page as PlaywrightPage } from 'playwright'; -import { forceClosePopup } from '../puppeteer/base-page'; -import { WebPage as PlaywrightWebPage } from './page'; - -const debug = getDebug('playwright:agent'); - -export class PlaywrightAgent extends PageAgent { - protected isRetryableContextError(error: unknown): boolean { - return isRetryableBrowserNavigationError(error); - } - - constructor(page: PlaywrightPage, opts?: WebPageAgentOpt) { - if (!page) { - throw new Error( - '[midscene] PlaywrightAgent requires a valid Playwright page instance. Please make sure to pass a valid page object.', - ); - } - const webPage = new PlaywrightWebPage(page, opts); - super(webPage, opts); - - const { forceSameTabNavigation = true, forceChromeSelectRendering } = - opts ?? {}; - - if (forceSameTabNavigation) { - forceClosePopup(page, debug); - } - - applyForceChromeSelectRendering( - page, - 'playwright', - forceChromeSelectRendering, - ); - } -} +export { + PlaywrightPageAgent, + PlaywrightPageAgent as PlaywrightAgent, +} from './page-agent'; diff --git a/packages/web-integration/src/playwright/browser-agent.ts b/packages/web-integration/src/playwright/browser-agent.ts index afcd37d3e5..f571a8803e 100644 --- a/packages/web-integration/src/playwright/browser-agent.ts +++ b/packages/web-integration/src/playwright/browser-agent.ts @@ -1,3 +1,7 @@ +import { + type BrowserAgentAdapter, + BrowserAgentPageController, +} from '@/common/browser-agent'; import { applyForceChromeSelectRendering, isRetryableBrowserNavigationError, @@ -13,6 +17,18 @@ import { WebPage as PlaywrightWebPage } from './page'; const debug = getDebug('playwright:browser-agent'); +const createPlaywrightBrowserAdapter = ( + context: PlaywrightBrowserContext, +): BrowserAgentAdapter => ({ + pages: () => context.pages(), + newPage: () => context.newPage(), + isPageClosed: (page) => page.isClosed(), + bringToFront: (page) => page.bringToFront(), + onNewPage: (handler) => context.on('page', handler), + offNewPage: (handler) => context.off('page', handler), + resolveNewPage: (page) => page, +}); + export type PlaywrightBrowserAgentOpt = Omit< WebPageAgentOpt, 'forceSameTabNavigation' @@ -26,13 +42,10 @@ export type PlaywrightBrowserAgentCreateOpt = PlaywrightBrowserAgentOpt & { }; export class PlaywrightBrowserAgent extends PageAgent { - private readonly context: PlaywrightBrowserContext; - private readonly autoFollowNewPage: boolean; - private readonly newPageTimeout: number; - - private readonly pageHandler = (page: PlaywrightPage) => { - void this.followPage(page); - }; + private readonly pageController: BrowserAgentPageController< + PlaywrightPage, + PlaywrightPage + >; protected isRetryableContextError(error: unknown): boolean { return isRetryableBrowserNavigationError(error); @@ -66,13 +79,17 @@ export class PlaywrightBrowserAgent extends PageAgent { }); super(webPage, agentOpts); - this.context = context; - this.autoFollowNewPage = autoFollowNewPage; - this.newPageTimeout = newPageTimeout; - - if (this.autoFollowNewPage) { - this.context.on('page', this.pageHandler); - } + this.pageController = new BrowserAgentPageController({ + agentName: 'PlaywrightBrowserAgent', + adapter: createPlaywrightBrowserAdapter(context), + getActivePage: () => this.interface.underlyingPage as PlaywrightPage, + setActivePageValue: (page) => { + this.interface.underlyingPage = page; + }, + autoFollowNewPage, + newPageTimeout, + debug, + }); applyForceChromeSelectRendering( initialPage, @@ -92,102 +109,30 @@ export class PlaywrightBrowserAgent extends PageAgent { } get activePage() { - return this.interface.underlyingPage as PlaywrightPage; + return this.pageController.activePage; } pages() { - return this.context.pages(); + return this.pageController.pages(); } async newPage() { - const page = await this.context.newPage(); - await this.setActivePage(page); - return page; + return this.pageController.newPage(); } async setActivePage(page: PlaywrightPage) { - if (!page || page.isClosed()) { - throw new Error( - '[midscene] Cannot set PlaywrightBrowserAgent active page to a closed or invalid page.', - ); - } - - this.interface.underlyingPage = page; - try { - await page.bringToFront(); - } catch (error) { - debug(`failed to bring page to front: ${error}`); - } + await this.pageController.setActivePage(page); } async waitForNewPage( action?: () => Promise | unknown, opts?: { timeout?: number }, ) { - const waiter = this.createNewPageWaiter(opts?.timeout); - - try { - await action?.(); - return await waiter.promise; - } catch (error) { - waiter.dispose(); - throw error; - } + return this.pageController.waitForNewPage(action, opts); } async destroy() { - this.context.off('page', this.pageHandler); + this.pageController.destroy(); await super.destroy(); } - - private async followPage(page: PlaywrightPage) { - try { - await this.setActivePage(page); - } catch (error) { - debug(`failed to follow new page: ${error}`); - } - } - - private createNewPageWaiter(timeout = this.newPageTimeout) { - let settled = false; - - const dispose = () => { - this.context.off('page', handler); - clearTimeout(timer); - }; - - const handler = (page: PlaywrightPage) => { - if (settled) { - return; - } - - settled = true; - dispose(); - resolvePage(page); - }; - - let resolvePage!: (page: PlaywrightPage) => void; - let rejectPage!: (error: unknown) => void; - const promise = new Promise((resolve, reject) => { - resolvePage = resolve; - rejectPage = reject; - }); - - const timer = setTimeout(() => { - if (settled) { - return; - } - settled = true; - dispose(); - rejectPage( - new Error( - `[midscene] Timed out waiting for a new Playwright page after ${timeout}ms.`, - ), - ); - }, timeout); - - this.context.on('page', handler); - - return { promise, dispose }; - } } diff --git a/packages/web-integration/src/playwright/index.ts b/packages/web-integration/src/playwright/index.ts index da45d28992..62a87d39d7 100644 --- a/packages/web-integration/src/playwright/index.ts +++ b/packages/web-integration/src/playwright/index.ts @@ -6,7 +6,7 @@ export { PlaywrightAiFixture } from './ai-fixture'; export { overrideAIConfig } from '@midscene/shared/env'; export { WebPage as PlaywrightWebPage } from './page'; export type { WebPageAgentOpt } from '@/web-element'; -export { PlaywrightAgent } from './agent'; +export { PlaywrightPageAgent, PlaywrightAgent } from './page-agent'; export { PlaywrightBrowserAgent, type PlaywrightBrowserAgentCreateOpt, diff --git a/packages/web-integration/src/playwright/page-agent.ts b/packages/web-integration/src/playwright/page-agent.ts new file mode 100644 index 0000000000..a096b23033 --- /dev/null +++ b/packages/web-integration/src/playwright/page-agent.ts @@ -0,0 +1,43 @@ +import { + applyForceChromeSelectRendering, + isRetryableBrowserNavigationError, +} from '@/common/web-agent'; +import type { WebPageAgentOpt } from '@/web-element'; +import { Agent as PageAgent } from '@midscene/core/agent'; +import { getDebug } from '@midscene/shared/logger'; +import type { Page as PlaywrightPage } from 'playwright'; +import { forceClosePopup } from '../puppeteer/base-page'; +import { WebPage as PlaywrightWebPage } from './page'; + +const debug = getDebug('playwright:agent'); + +export class PlaywrightPageAgent extends PageAgent { + protected isRetryableContextError(error: unknown): boolean { + return isRetryableBrowserNavigationError(error); + } + + constructor(page: PlaywrightPage, opts?: WebPageAgentOpt) { + if (!page) { + throw new Error( + '[midscene] PlaywrightPageAgent requires a valid Playwright page instance. Please make sure to pass a valid page object.', + ); + } + const webPage = new PlaywrightWebPage(page, opts); + super(webPage, opts); + + const { forceSameTabNavigation = true, forceChromeSelectRendering } = + opts ?? {}; + + if (forceSameTabNavigation) { + forceClosePopup(page, debug); + } + + applyForceChromeSelectRendering( + page, + 'playwright', + forceChromeSelectRendering, + ); + } +} + +export { PlaywrightPageAgent as PlaywrightAgent }; diff --git a/packages/web-integration/src/puppeteer/agent-launcher.ts b/packages/web-integration/src/puppeteer/agent-launcher.ts index 200716cda7..51dd19c58a 100644 --- a/packages/web-integration/src/puppeteer/agent-launcher.ts +++ b/packages/web-integration/src/puppeteer/agent-launcher.ts @@ -389,10 +389,26 @@ export async function puppeteerAgentForTarget( typeof target.forceSameTabNavigation !== 'undefined' ? target.forceSameTabNavigation : true; + const mode = target.mode ?? 'page'; - if (target.autoFollowNewPage && target.forceSameTabNavigation === true) { + if (mode !== 'page' && mode !== 'browser') { throw new Error( - '[midscene] autoFollowNewPage cannot be used with forceSameTabNavigation: true.', + `[midscene] web target mode must be either "page" or "browser", but got "${mode}".`, + ); + } + + if (mode === 'page' && target.autoFollowNewPage) { + throw new Error( + '[midscene] autoFollowNewPage requires browser mode. Use browser: or web.mode: browser.', + ); + } + + if ( + mode === 'browser' && + typeof target.forceSameTabNavigation !== 'undefined' + ) { + throw new Error( + '[midscene] forceSameTabNavigation cannot be used in browser mode. Use page: or web.mode: page when same-tab navigation is required.', ); } @@ -406,16 +422,17 @@ export async function puppeteerAgentForTarget( }; // prepare Midscene agent - const agent = target.autoFollowNewPage - ? await PuppeteerBrowserAgent.create(page.browser(), { - ...commonAgentOpts, - initialPage: page, - autoFollowNewPage: true, - }) - : new PuppeteerAgent(page, { - ...commonAgentOpts, - forceSameTabNavigation, - }); + const agent = + mode === 'browser' + ? await PuppeteerBrowserAgent.create(page.browser(), { + ...commonAgentOpts, + initialPage: page, + autoFollowNewPage: target.autoFollowNewPage ?? false, + }) + : new PuppeteerAgent(page, { + ...commonAgentOpts, + forceSameTabNavigation, + }); freeFn.push({ name: 'midscene_puppeteer_agent', diff --git a/packages/web-integration/src/puppeteer/agent.ts b/packages/web-integration/src/puppeteer/agent.ts index 3a5685ea16..45e94ec026 100644 --- a/packages/web-integration/src/puppeteer/agent.ts +++ b/packages/web-integration/src/puppeteer/agent.ts @@ -1,41 +1,4 @@ -import { - applyForceChromeSelectRendering, - isRetryableBrowserNavigationError, -} from '@/common/web-agent'; -import type { WebPageAgentOpt } from '@/web-element'; -import { Agent as PageAgent } from '@midscene/core/agent'; -import { getDebug } from '@midscene/shared/logger'; -import type { Page as PuppeteerPage } from 'puppeteer'; -import { forceClosePopup } from './base-page'; -import { PuppeteerWebPage } from './page'; - -const debug = getDebug('puppeteer:agent'); - -export class PuppeteerAgent extends PageAgent { - protected isRetryableContextError(error: unknown): boolean { - return isRetryableBrowserNavigationError(error); - } - - constructor(page: PuppeteerPage, opts?: WebPageAgentOpt) { - if (!page) { - throw new Error( - '[midscene] PuppeteerAgent requires a valid Puppeteer page instance. Please make sure to pass a valid page object.', - ); - } - const webPage = new PuppeteerWebPage(page, opts); - super(webPage, opts); - - const { forceSameTabNavigation = true, forceChromeSelectRendering } = - opts ?? {}; - - if (forceSameTabNavigation) { - forceClosePopup(page, debug); - } - - applyForceChromeSelectRendering( - page, - 'puppeteer', - forceChromeSelectRendering, - ); - } -} +export { + PuppeteerPageAgent, + PuppeteerPageAgent as PuppeteerAgent, +} from './page-agent'; diff --git a/packages/web-integration/src/puppeteer/browser-agent.ts b/packages/web-integration/src/puppeteer/browser-agent.ts index 5f4e72d3d3..0f5e482cbf 100644 --- a/packages/web-integration/src/puppeteer/browser-agent.ts +++ b/packages/web-integration/src/puppeteer/browser-agent.ts @@ -1,3 +1,7 @@ +import { + type BrowserAgentAdapter, + BrowserAgentPageController, +} from '@/common/browser-agent'; import { applyForceChromeSelectRendering, isRetryableBrowserNavigationError, @@ -14,6 +18,19 @@ import { PuppeteerWebPage } from './page'; const debug = getDebug('puppeteer:browser-agent'); +const createPuppeteerBrowserAdapter = ( + browser: PuppeteerBrowser, +): BrowserAgentAdapter => ({ + pages: () => browser.pages(), + newPage: () => browser.newPage(), + isPageClosed: (page) => page.isClosed(), + bringToFront: (page) => page.bringToFront(), + onNewPage: (handler) => browser.on('targetcreated', handler), + offNewPage: (handler) => browser.off('targetcreated', handler), + isNewPageEvent: (target) => target.type() === 'page', + resolveNewPage: (target) => target.page(), +}); + export type PuppeteerBrowserAgentOpt = Omit< WebPageAgentOpt, 'forceSameTabNavigation' @@ -27,13 +44,10 @@ export type PuppeteerBrowserAgentCreateOpt = PuppeteerBrowserAgentOpt & { }; export class PuppeteerBrowserAgent extends PageAgent { - private readonly browser: PuppeteerBrowser; - private readonly autoFollowNewPage: boolean; - private readonly newPageTimeout: number; - - private readonly targetCreatedHandler = (target: PuppeteerTarget) => { - void this.followTarget(target); - }; + private readonly pageController: BrowserAgentPageController< + PuppeteerPage, + PuppeteerTarget + >; protected isRetryableContextError(error: unknown): boolean { return isRetryableBrowserNavigationError(error); @@ -67,13 +81,17 @@ export class PuppeteerBrowserAgent extends PageAgent { }); super(webPage, agentOpts); - this.browser = browser; - this.autoFollowNewPage = autoFollowNewPage; - this.newPageTimeout = newPageTimeout; - - if (this.autoFollowNewPage) { - this.browser.on('targetcreated', this.targetCreatedHandler); - } + this.pageController = new BrowserAgentPageController({ + agentName: 'PuppeteerBrowserAgent', + adapter: createPuppeteerBrowserAdapter(browser), + getActivePage: () => this.interface.underlyingPage as PuppeteerPage, + setActivePageValue: (page) => { + this.interface.underlyingPage = page; + }, + autoFollowNewPage, + newPageTimeout, + debug, + }); applyForceChromeSelectRendering( initialPage, @@ -94,118 +112,30 @@ export class PuppeteerBrowserAgent extends PageAgent { } get activePage() { - return this.interface.underlyingPage as PuppeteerPage; + return this.pageController.activePage; } - async pages() { - return this.browser.pages(); + pages() { + return this.pageController.pages(); } async newPage() { - const page = await this.browser.newPage(); - await this.setActivePage(page); - return page; + return this.pageController.newPage(); } async setActivePage(page: PuppeteerPage) { - if (!page || page.isClosed()) { - throw new Error( - '[midscene] Cannot set PuppeteerBrowserAgent active page to a closed or invalid page.', - ); - } - - this.interface.underlyingPage = page; - try { - await page.bringToFront(); - } catch (error) { - debug(`failed to bring page to front: ${error}`); - } + await this.pageController.setActivePage(page); } async waitForNewPage( action?: () => Promise | unknown, opts?: { timeout?: number }, ) { - const waiter = this.createNewPageWaiter(opts?.timeout); - - try { - await action?.(); - return await waiter.promise; - } catch (error) { - waiter.dispose(); - throw error; - } + return this.pageController.waitForNewPage(action, opts); } async destroy() { - this.browser.off('targetcreated', this.targetCreatedHandler); + this.pageController.destroy(); await super.destroy(); } - - private async followTarget(target: PuppeteerTarget) { - if (target.type() !== 'page') { - return; - } - - try { - const page = await target.page(); - if (page) { - await this.setActivePage(page); - } - } catch (error) { - debug(`failed to follow new page: ${error}`); - } - } - - private createNewPageWaiter(timeout = this.newPageTimeout) { - let settled = false; - - const dispose = () => { - this.browser.off('targetcreated', handler); - clearTimeout(timer); - }; - - const handler = async (target: PuppeteerTarget) => { - if (target.type() !== 'page' || settled) { - return; - } - - settled = true; - dispose(); - - try { - const page = await target.page(); - if (!page) { - throw new Error('new target did not resolve to a page'); - } - resolvePage(page); - } catch (error) { - rejectPage(error); - } - }; - - let resolvePage!: (page: PuppeteerPage) => void; - let rejectPage!: (error: unknown) => void; - const promise = new Promise((resolve, reject) => { - resolvePage = resolve; - rejectPage = reject; - }); - - const timer = setTimeout(() => { - if (settled) { - return; - } - settled = true; - dispose(); - rejectPage( - new Error( - `[midscene] Timed out waiting for a new Puppeteer page after ${timeout}ms.`, - ), - ); - }, timeout); - - this.browser.on('targetcreated', handler); - - return { promise, dispose }; - } } diff --git a/packages/web-integration/src/puppeteer/index.ts b/packages/web-integration/src/puppeteer/index.ts index 0f0b662a9b..c68f24e13f 100644 --- a/packages/web-integration/src/puppeteer/index.ts +++ b/packages/web-integration/src/puppeteer/index.ts @@ -1,4 +1,4 @@ -export { PuppeteerAgent } from './agent'; +export { PuppeteerPageAgent, PuppeteerAgent } from './page-agent'; export { PuppeteerBrowserAgent, type PuppeteerBrowserAgentCreateOpt, diff --git a/packages/web-integration/src/puppeteer/page-agent.ts b/packages/web-integration/src/puppeteer/page-agent.ts new file mode 100644 index 0000000000..45ebbf10f2 --- /dev/null +++ b/packages/web-integration/src/puppeteer/page-agent.ts @@ -0,0 +1,43 @@ +import { + applyForceChromeSelectRendering, + isRetryableBrowserNavigationError, +} from '@/common/web-agent'; +import type { WebPageAgentOpt } from '@/web-element'; +import { Agent as PageAgent } from '@midscene/core/agent'; +import { getDebug } from '@midscene/shared/logger'; +import type { Page as PuppeteerPage } from 'puppeteer'; +import { forceClosePopup } from './base-page'; +import { PuppeteerWebPage } from './page'; + +const debug = getDebug('puppeteer:agent'); + +export class PuppeteerPageAgent extends PageAgent { + protected isRetryableContextError(error: unknown): boolean { + return isRetryableBrowserNavigationError(error); + } + + constructor(page: PuppeteerPage, opts?: WebPageAgentOpt) { + if (!page) { + throw new Error( + '[midscene] PuppeteerPageAgent requires a valid Puppeteer page instance. Please make sure to pass a valid page object.', + ); + } + const webPage = new PuppeteerWebPage(page, opts); + super(webPage, opts); + + const { forceSameTabNavigation = true, forceChromeSelectRendering } = + opts ?? {}; + + if (forceSameTabNavigation) { + forceClosePopup(page, debug); + } + + applyForceChromeSelectRendering( + page, + 'puppeteer', + forceChromeSelectRendering, + ); + } +} + +export { PuppeteerPageAgent as PuppeteerAgent }; diff --git a/packages/web-integration/tests/unit-test/browser-agent-page-controller.test.ts b/packages/web-integration/tests/unit-test/browser-agent-page-controller.test.ts new file mode 100644 index 0000000000..d8412c06a1 --- /dev/null +++ b/packages/web-integration/tests/unit-test/browser-agent-page-controller.test.ts @@ -0,0 +1,122 @@ +import { BrowserAgentPageController } from '@/common/browser-agent'; +import { describe, expect, it, vi } from 'vitest'; + +type PageMock = { + id: string; + closed?: boolean; + bringToFront: ReturnType; +}; + +type NewPageEvent = { + kind: 'page' | 'worker'; + page?: PageMock | null; +}; + +const createPage = (id: string): PageMock => ({ + id, + bringToFront: vi.fn(), +}); + +function createController(options?: { + autoFollowNewPage?: boolean; + newPage?: PageMock; +}) { + let activePage = createPage('initial'); + const handlers = new Set<(event: NewPageEvent) => void>(); + const debug = vi.fn(); + const newPage = options?.newPage ?? createPage('created'); + + const controller = new BrowserAgentPageController({ + agentName: 'TestBrowserAgent', + autoFollowNewPage: options?.autoFollowNewPage ?? false, + newPageTimeout: 50, + debug, + getActivePage: () => activePage, + setActivePageValue: (page) => { + activePage = page; + }, + adapter: { + pages: () => [activePage], + newPage: async () => newPage, + isPageClosed: (page) => Boolean(page.closed), + bringToFront: (page) => page.bringToFront(), + onNewPage: (handler) => { + handlers.add(handler); + }, + offNewPage: (handler) => { + handlers.delete(handler); + }, + isNewPageEvent: (event) => event.kind === 'page', + resolveNewPage: (event) => event.page ?? null, + }, + }); + + return { + controller, + get activePage() { + return activePage; + }, + emit: (event: NewPageEvent) => { + for (const handler of handlers) { + handler(event); + } + }, + handlers, + debug, + newPage, + }; +} + +describe('BrowserAgentPageController', () => { + it('sets the created page as active page', async () => { + const ctx = createController(); + + const page = await ctx.controller.newPage(); + + expect(page.id).toBe('created'); + expect(ctx.activePage).toBe(page); + expect(page.bringToFront).toHaveBeenCalledTimes(1); + }); + + it('auto-follows matching new page events', async () => { + const ctx = createController({ autoFollowNewPage: true }); + const nextPage = createPage('next'); + + ctx.emit({ kind: 'worker' }); + expect(ctx.activePage.id).toBe('initial'); + + ctx.emit({ kind: 'page', page: nextPage }); + await vi.waitFor(() => expect(ctx.activePage).toBe(nextPage)); + expect(nextPage.bringToFront).toHaveBeenCalledTimes(1); + }); + + it('waits for the next page without switching active page', async () => { + const ctx = createController(); + const nextPage = createPage('next'); + + const waiting = ctx.controller.waitForNewPage(); + ctx.emit({ kind: 'worker' }); + ctx.emit({ kind: 'page', page: nextPage }); + + await expect(waiting).resolves.toBe(nextPage); + expect(ctx.activePage.id).toBe('initial'); + }); + + it('removes the auto-follow listener on destroy', () => { + const ctx = createController({ autoFollowNewPage: true }); + + expect(ctx.handlers.size).toBe(1); + ctx.controller.destroy(); + expect(ctx.handlers.size).toBe(0); + }); + + it('rejects closed pages', async () => { + const ctx = createController(); + const closedPage = createPage('closed'); + closedPage.closed = true; + + await expect(ctx.controller.setActivePage(closedPage)).rejects.toThrow( + '[midscene] Cannot set TestBrowserAgent active page to a closed or invalid page.', + ); + }); +}); diff --git a/packages/web-integration/tests/unit-test/circular-dependency.test.ts b/packages/web-integration/tests/unit-test/circular-dependency.test.ts index f5822b0017..4472a5623a 100644 --- a/packages/web-integration/tests/unit-test/circular-dependency.test.ts +++ b/packages/web-integration/tests/unit-test/circular-dependency.test.ts @@ -66,11 +66,19 @@ describe('circular dependency detection', () => { it('should properly export all public APIs', async () => { // This test verifies that all expected exports are available - const { PlaywrightAgent, PuppeteerAgent, PageAgent, StaticPageAgent } = - await import('@midscene/web'); + const { + PlaywrightAgent, + PlaywrightPageAgent, + PuppeteerAgent, + PuppeteerPageAgent, + PageAgent, + StaticPageAgent, + } = await import('@midscene/web'); expect(PlaywrightAgent).toBeDefined(); + expect(PlaywrightPageAgent).toBe(PlaywrightAgent); expect(PuppeteerAgent).toBeDefined(); + expect(PuppeteerPageAgent).toBe(PuppeteerAgent); expect(PageAgent).toBeDefined(); expect(StaticPageAgent).toBeDefined(); }); diff --git a/packages/web-integration/tests/unit-test/constructor-validation.test.ts b/packages/web-integration/tests/unit-test/constructor-validation.test.ts index c5f4de30e8..914caeb891 100644 --- a/packages/web-integration/tests/unit-test/constructor-validation.test.ts +++ b/packages/web-integration/tests/unit-test/constructor-validation.test.ts @@ -1,33 +1,45 @@ import { describe, expect, it } from 'vitest'; describe('PlaywrightAgent constructor validation', () => { + it('should keep PlaywrightAgent as an alias of PlaywrightPageAgent', async () => { + const { PlaywrightAgent, PlaywrightPageAgent } = await import( + '@/playwright' + ); + expect(PlaywrightAgent).toBe(PlaywrightPageAgent); + }); + it('should throw when page is undefined', async () => { const { PlaywrightAgent } = await import('@/playwright'); expect(() => new PlaywrightAgent(undefined as any)).toThrow( - '[midscene] PlaywrightAgent requires a valid Playwright page instance', + '[midscene] PlaywrightPageAgent requires a valid Playwright page instance', ); }); it('should throw when page is null', async () => { const { PlaywrightAgent } = await import('@/playwright'); expect(() => new PlaywrightAgent(null as any)).toThrow( - '[midscene] PlaywrightAgent requires a valid Playwright page instance', + '[midscene] PlaywrightPageAgent requires a valid Playwright page instance', ); }); }); describe('PuppeteerAgent constructor validation', () => { + it('should keep PuppeteerAgent as an alias of PuppeteerPageAgent', async () => { + const { PuppeteerAgent, PuppeteerPageAgent } = await import('@/puppeteer'); + expect(PuppeteerAgent).toBe(PuppeteerPageAgent); + }); + it('should throw when page is undefined', async () => { const { PuppeteerAgent } = await import('@/puppeteer'); expect(() => new PuppeteerAgent(undefined as any)).toThrow( - '[midscene] PuppeteerAgent requires a valid Puppeteer page instance', + '[midscene] PuppeteerPageAgent requires a valid Puppeteer page instance', ); }); it('should throw when page is null', async () => { const { PuppeteerAgent } = await import('@/puppeteer'); expect(() => new PuppeteerAgent(null as any)).toThrow( - '[midscene] PuppeteerAgent requires a valid Puppeteer page instance', + '[midscene] PuppeteerPageAgent requires a valid Puppeteer page instance', ); }); }); diff --git a/packages/web-integration/tests/unit-test/puppeteer/agent-launcher.test.ts b/packages/web-integration/tests/unit-test/puppeteer/agent-launcher.test.ts index ea66069e46..f4c2646373 100644 --- a/packages/web-integration/tests/unit-test/puppeteer/agent-launcher.test.ts +++ b/packages/web-integration/tests/unit-test/puppeteer/agent-launcher.test.ts @@ -15,6 +15,8 @@ const mockNewPage = vi.fn(); let pageMock: ReturnType; const browserMock = { newPage: mockNewPage, + on: vi.fn(), + off: vi.fn(), setCookie: vi.fn(), close: vi.fn(), }; @@ -25,6 +27,8 @@ const createPageMock = () => ({ setViewport: vi.fn().mockResolvedValue(undefined), goto: vi.fn().mockResolvedValue(undefined), waitForNetworkIdle: vi.fn().mockResolvedValue(undefined), + browser: vi.fn(() => browserMock), + bringToFront: vi.fn().mockResolvedValue(undefined), on: vi.fn(), isClosed: vi.fn().mockReturnValue(false), }); @@ -222,4 +226,37 @@ describe('launchPuppeteerPage', () => { expect((agent.page as any).waitForNetworkIdleTimeout).toBe(4321); }); + + it('requires browser mode for autoFollowNewPage', async () => { + await expect( + puppeteerAgentForTarget({ + url: 'https://example.com', + autoFollowNewPage: true, + }), + ).rejects.toThrow('autoFollowNewPage requires browser mode'); + }); + + it('creates browser agent in browser mode', async () => { + const { agent } = await puppeteerAgentForTarget({ + mode: 'browser', + url: 'https://example.com', + autoFollowNewPage: true, + }); + + expect(agent.constructor.name).toBe('PuppeteerBrowserAgent'); + expect(browserMock.on).toHaveBeenCalledWith( + 'targetcreated', + expect.any(Function), + ); + }); + + it('rejects forceSameTabNavigation in browser mode', async () => { + await expect( + puppeteerAgentForTarget({ + mode: 'browser', + url: 'https://example.com', + forceSameTabNavigation: false, + }), + ).rejects.toThrow('forceSameTabNavigation cannot be used in browser mode'); + }); }); diff --git a/packages/web-integration/tests/unit-test/yaml/__snapshots__/utils.test.ts.snap b/packages/web-integration/tests/unit-test/yaml/__snapshots__/utils.test.ts.snap index 48b2d3db29..2035f195cf 100644 --- a/packages/web-integration/tests/unit-test/yaml/__snapshots__/utils.test.ts.snap +++ b/packages/web-integration/tests/unit-test/yaml/__snapshots__/utils.test.ts.snap @@ -1,7 +1,7 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html exports[`utils > build yaml 1`] = ` -"target: +"page: url: https://www.example.com tasks: [] " diff --git a/packages/web-integration/tests/unit-test/yaml/utils.test.ts b/packages/web-integration/tests/unit-test/yaml/utils.test.ts index dbb064bd8e..073e14d75c 100644 --- a/packages/web-integration/tests/unit-test/yaml/utils.test.ts +++ b/packages/web-integration/tests/unit-test/yaml/utils.test.ts @@ -1,4 +1,8 @@ -import { buildYaml, parseYamlScript } from '@midscene/core/yaml'; +import { + buildYaml, + parseYamlScript, + resolveWebTarget, +} from '@midscene/core/yaml'; import { describe, expect, test } from 'vitest'; describe('utils', () => { @@ -138,5 +142,98 @@ tasks: const result = parseYamlScript(yamlContent); expect(result).toMatchSnapshot(); }); + + test('supports explicit page target', () => { + const yamlContent = ` +page: + url: "https://example.com" +tasks: +- sleep: 1000 +`; + + const result = parseYamlScript(yamlContent); + const resolvedTarget = resolveWebTarget(result); + + expect(resolvedTarget?.source).toBe('page'); + expect(resolvedTarget?.mode).toBe('page'); + expect(resolvedTarget?.target.url).toBe('https://example.com'); + }); + + test('supports explicit browser target', () => { + const yamlContent = ` +browser: + url: "https://example.com" + autoFollowNewPage: true +tasks: +- sleep: 1000 +`; + + const result = parseYamlScript(yamlContent); + const resolvedTarget = resolveWebTarget(result); + + expect(resolvedTarget?.source).toBe('browser'); + expect(resolvedTarget?.mode).toBe('browser'); + expect(resolvedTarget?.target.autoFollowNewPage).toBe(true); + }); + + test('supports web mode browser compatibility target', () => { + const yamlContent = ` +web: + mode: browser + url: "https://example.com" + autoFollowNewPage: true +tasks: +- sleep: 1000 +`; + + const result = parseYamlScript(yamlContent); + const resolvedTarget = resolveWebTarget(result); + + expect(resolvedTarget?.source).toBe('web'); + expect(resolvedTarget?.mode).toBe('browser'); + }); + + test('rejects multiple web targets', () => { + const yamlContent = ` +page: + url: "https://example.com" +browser: + url: "https://example.com" +tasks: +- sleep: 1000 +`; + + expect(() => parseYamlScript(yamlContent)).toThrow( + 'Only one web target can be specified', + ); + }); + + test('rejects implicit browser mode from web autoFollowNewPage', () => { + const yamlContent = ` +web: + url: "https://example.com" + autoFollowNewPage: true +tasks: +- sleep: 1000 +`; + + expect(() => parseYamlScript(yamlContent)).toThrow( + 'autoFollowNewPage requires browser mode', + ); + }); + + test('rejects forceSameTabNavigation in browser mode', () => { + const yamlContent = ` +browser: + url: "https://example.com" + forceSameTabNavigation: false +tasks: +- sleep: 1000 +`; + + expect(() => parseYamlScript(yamlContent)).toThrow( + 'forceSameTabNavigation cannot be used in browser mode', + ); + }); }); }); From 3155eb9db3eb61b7d631d1049c24bd0ed7a58a87 Mon Sep 17 00:00:00 2001 From: quanru Date: Thu, 11 Jun 2026 14:21:43 +0800 Subject: [PATCH 4/7] fix(workflow): satisfy test type checks --- .../tests/unit-test/config-factory.test.ts | 4 ++++ .../ai/web/puppeteer/tab-navigation.test.ts | 20 +++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/packages/cli/tests/unit-test/config-factory.test.ts b/packages/cli/tests/unit-test/config-factory.test.ts index ffd3388829..ccddf18118 100644 --- a/packages/cli/tests/unit-test/config-factory.test.ts +++ b/packages/cli/tests/unit-test/config-factory.test.ts @@ -272,8 +272,12 @@ concurrent: 2 dotenvDebug: true, summary: 'parsed.json', shareBrowserContext: false, + page: undefined, + browser: undefined, web: { userAgent: 'from-file', viewportWidth: 800 }, + target: undefined, android: { deviceId: 'from-file' }, + ios: undefined, }; vi.mocked(readFileSync).mockReturnValue(mockYamlContent); vi.mocked(yamlLoad).mockReturnValue(mockParsedYaml); diff --git a/packages/web-integration/tests/ai/web/puppeteer/tab-navigation.test.ts b/packages/web-integration/tests/ai/web/puppeteer/tab-navigation.test.ts index dbf8ed02f2..5c827a7ec4 100644 --- a/packages/web-integration/tests/ai/web/puppeteer/tab-navigation.test.ts +++ b/packages/web-integration/tests/ai/web/puppeteer/tab-navigation.test.ts @@ -9,11 +9,21 @@ import { } from './test-utils'; import { launchPage } from './utils'; -const clickNewTabLink = async (page: PuppeteerPage) => { - const popupPromise = new Promise((resolve) => { - page.once('popup', resolve); +const waitForPopup = (page: PuppeteerPage) => + new Promise((resolve, reject) => { + page.once('popup', (popup) => { + if (!popup) { + reject(new Error('Expected popup page to be available')); + return; + } + + resolve(popup); + }); }); +const clickNewTabLink = async (page: PuppeteerPage) => { + const popupPromise = waitForPopup(page); + await page.click('#newTabLink'); const popup = await popupPromise; await popup.waitForSelector('.weather-container'); @@ -24,9 +34,7 @@ const tapNewTabLinkWithAgent = async ( agent: PuppeteerAgent | PuppeteerBrowserAgent, page: PuppeteerPage, ) => { - const popupPromise = new Promise((resolve) => { - page.once('popup', resolve); - }); + const popupPromise = waitForPopup(page); await agent.aiTap('the "Open in New Tab" link on the original page', { xpath: '//*[@id="newTabLink"]', From e25fde234d3b82a93dc3a380550591f01fc1f63d Mon Sep 17 00:00:00 2001 From: quanru Date: Thu, 11 Jun 2026 14:36:02 +0800 Subject: [PATCH 5/7] fix(cli): preserve yaml mock exports --- .../cli/tests/unit-test/create-yaml-player.test.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/cli/tests/unit-test/create-yaml-player.test.ts b/packages/cli/tests/unit-test/create-yaml-player.test.ts index 208159f65a..5ddc4dbed9 100644 --- a/packages/cli/tests/unit-test/create-yaml-player.test.ts +++ b/packages/cli/tests/unit-test/create-yaml-player.test.ts @@ -22,10 +22,14 @@ vi.mock('http-server', () => ({ createServer: vi.fn(), })); -vi.mock('@midscene/core/yaml', () => ({ - ScriptPlayer: vi.fn(), - parseYamlScript: vi.fn(), -})); +vi.mock('@midscene/core/yaml', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + ScriptPlayer: vi.fn(), + parseYamlScript: vi.fn(), + }; +}); vi.mock('@midscene/core/agent', () => ({ createAgent: vi.fn(), From d550893eaccbcb47d58dcf71e2d58e0d631d35db Mon Sep 17 00:00:00 2001 From: quanru Date: Thu, 11 Jun 2026 14:47:37 +0800 Subject: [PATCH 6/7] test(web-integration): update yaml page snapshot --- .../tests/unit-test/yaml/__snapshots__/player.test.ts.snap | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/web-integration/tests/unit-test/yaml/__snapshots__/player.test.ts.snap b/packages/web-integration/tests/unit-test/yaml/__snapshots__/player.test.ts.snap index c4b69abc91..9e487246d9 100644 --- a/packages/web-integration/tests/unit-test/yaml/__snapshots__/player.test.ts.snap +++ b/packages/web-integration/tests/unit-test/yaml/__snapshots__/player.test.ts.snap @@ -1,7 +1,7 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html exports[`yaml utils > basic build && load 1`] = ` -"target: +"page: url: https://bing.com waitForNetworkIdle: timeout: 1000 @@ -15,7 +15,7 @@ tasks: exports[`yaml utils > basic build && load 2`] = ` { - "target": { + "page": { "url": "https://bing.com", "waitForNetworkIdle": { "continueOnNetworkIdleError": true, From d26cf55d30163aad57b08cb96a292563ed8c021b Mon Sep 17 00:00:00 2001 From: quanru Date: Mon, 22 Jun 2026 14:31:22 +0800 Subject: [PATCH 7/7] refactor(web-integration): share browser agent runtime modes --- apps/site/docs/en/web-api-reference.mdx | 2 + apps/site/docs/zh/web-api-reference.mdx | 2 + .../src/common/browser-agent.ts | 92 +++++++++++++++++++ .../src/playwright/ai-fixture.ts | 14 ++- .../src/playwright/browser-agent.ts | 45 +++++---- .../src/playwright/page-agent.ts | 21 ++++- .../src/puppeteer/agent-launcher.ts | 29 ++---- .../src/puppeteer/browser-agent.ts | 45 +++++---- .../src/puppeteer/page-agent.ts | 26 ++++-- .../browser-agent-page-controller.test.ts | 55 ++++++++++- 10 files changed, 261 insertions(+), 70 deletions(-) diff --git a/apps/site/docs/en/web-api-reference.mdx b/apps/site/docs/en/web-api-reference.mdx index 88da996304..26be54807c 100644 --- a/apps/site/docs/en/web-api-reference.mdx +++ b/apps/site/docs/en/web-api-reference.mdx @@ -59,6 +59,7 @@ In addition to the base agent options, Puppeteer exposes: :::info - One agent per page: by default (`forceSameTabNavigation: true`), Midscene opens new links in the current tab for easier debugging. Set it to `false` if you want normal new-tab behavior and create a new `PuppeteerAgent` for each page yourself. Use `PuppeteerBrowserAgent` when the same Agent should manage browser-level page switching. +- `PuppeteerAgent` / `PuppeteerPageAgent` remains page-scoped for compatibility. It does not expose browser-level page switching unless you explicitly choose `PuppeteerBrowserAgent`. - For the full list of interaction methods, see [API reference (Common)](./api#interaction-methods). ::: @@ -169,6 +170,7 @@ const agent = new PlaywrightPageAgent(page, { :::info - One agent per page: with `forceSameTabNavigation` (default `true`), Midscene intercepts new tabs for stability. Set it to `false` to allow normal new tabs and create a new `PlaywrightAgent` for each page yourself. Use `PlaywrightBrowserAgent` when the same Agent should manage browser-context-level page switching. +- `PlaywrightAgent` / `PlaywrightPageAgent` remains page-scoped for compatibility. It does not expose browser-level page switching unless you explicitly choose `PlaywrightBrowserAgent`. - For the full list of interaction methods, see [API reference (Common)](./api#interaction-methods). ::: diff --git a/apps/site/docs/zh/web-api-reference.mdx b/apps/site/docs/zh/web-api-reference.mdx index 3eeebfbd50..aa44895ab9 100644 --- a/apps/site/docs/zh/web-api-reference.mdx +++ b/apps/site/docs/zh/web-api-reference.mdx @@ -59,6 +59,7 @@ const agent = new PuppeteerPageAgent(page, { :::info - 每个页面一个 Agent：默认情况下（`forceSameTabNavigation: true`）Midscene 会拦截新标签并在当前页打开，便于调试；若想保留浏览器原生的新标签行为可设为 `false`，并自行给每个页面创建新的 `PuppeteerAgent`。如果需要同一个 Agent 管理浏览器级别的页面切换，请使用 `PuppeteerBrowserAgent`。 +- `PuppeteerAgent` / `PuppeteerPageAgent` 为了兼容性仍保持 page-scoped 语义，不会暴露浏览器级别的页面切换能力；需要时请显式选择 `PuppeteerBrowserAgent`。 - 更多交互方法请参考 [API 参考（通用）](./api#interaction-methods)。 ::: @@ -169,6 +170,7 @@ const agent = new PlaywrightPageAgent(page, { :::info - 每个页面一个 Agent：默认 `forceSameTabNavigation` 为 `true`，Midscene 会拦截新标签确保稳定性；如需浏览器原生新标签行为请设为 `false`，并自行给每个页面创建新的 `PlaywrightAgent`。如果需要同一个 Agent 管理 browser context 级别的页面切换，请使用 `PlaywrightBrowserAgent`。 +- `PlaywrightAgent` / `PlaywrightPageAgent` 为了兼容性仍保持 page-scoped 语义，不会暴露浏览器级别的页面切换能力；需要时请显式选择 `PlaywrightBrowserAgent`。 - 更多交互方法请参考 [API 参考（通用）](./api#interaction-methods)。 ::: diff --git a/packages/web-integration/src/common/browser-agent.ts b/packages/web-integration/src/common/browser-agent.ts index 2de246ab4c..a051ef1a6c 100644 --- a/packages/web-integration/src/common/browser-agent.ts +++ b/packages/web-integration/src/common/browser-agent.ts @@ -1,5 +1,10 @@ +import type { AgentOpt } from '@midscene/core'; +import { Agent as CoreAgent } from '@midscene/core/agent'; +import type { AbstractInterface } from '@midscene/core/device'; import type { DebugFunction } from '@midscene/shared/logger'; +export type BrowserAgentPageScope = 'page' | 'browser'; + export type BrowserAgentAdapter = { pages(): Page[] | Promise; newPage(): Promise; @@ -21,6 +26,93 @@ export type BrowserAgentPageControllerOptions = { debug: DebugFunction; }; +export type BrowserAgentRuntimeOptions = { + agentName: string; + pageScope: BrowserAgentPageScope; + forceSameTabNavigation?: boolean; + autoFollowNewPage?: boolean; + newPageTimeout?: number; +}; + +export type ResolvedBrowserAgentRuntimeOptions = { + pageScope: BrowserAgentPageScope; + forceSameTabNavigation: boolean; + autoFollowNewPage: boolean; + newPageTimeout: number; +}; + +const DEFAULT_NEW_PAGE_TIMEOUT = 5000; + +export function resolveBrowserAgentRuntimeOptions({ + agentName, + pageScope, + forceSameTabNavigation, + autoFollowNewPage, + newPageTimeout = DEFAULT_NEW_PAGE_TIMEOUT, +}: BrowserAgentRuntimeOptions): ResolvedBrowserAgentRuntimeOptions { + if (pageScope === 'page') { + if (autoFollowNewPage) { + throw new Error( + `[midscene] autoFollowNewPage requires browser mode for ${agentName}. Use BrowserAgent when one agent should follow newly opened pages.`, + ); + } + + return { + pageScope, + forceSameTabNavigation: forceSameTabNavigation ?? true, + autoFollowNewPage: false, + newPageTimeout, + }; + } + + if (typeof forceSameTabNavigation !== 'undefined') { + throw new Error( + `[midscene] forceSameTabNavigation cannot be used in browser mode for ${agentName}. Use PageAgent when same-tab navigation is required.`, + ); + } + + return { + pageScope, + forceSameTabNavigation: false, + autoFollowNewPage: autoFollowNewPage ?? false, + newPageTimeout, + }; +} + +export abstract class BrowserAwareAgent< + InterfaceType extends AbstractInterface, + Page, + NewPageEvent, +> extends CoreAgent { + private readonly browserPageController?: BrowserAgentPageController< + Page, + NewPageEvent + >; + + protected constructor( + interfaceInstance: InterfaceType, + opts?: AgentOpt, + pageController?: BrowserAgentPageController, + ) { + super(interfaceInstance, opts); + this.browserPageController = pageController; + } + + protected getPageController() { + if (!this.browserPageController) { + throw new Error( + `[midscene] ${this.constructor.name} is running in page mode and cannot control browser pages.`, + ); + } + return this.browserPageController; + } + + async destroy() { + this.browserPageController?.destroy(); + await super.destroy(); + } +} + export class BrowserAgentPageController { private readonly agentName: string; private readonly adapter: BrowserAgentAdapter; diff --git a/packages/web-integration/src/playwright/ai-fixture.ts b/packages/web-integration/src/playwright/ai-fixture.ts index 752431f175..9e40d767e1 100644 --- a/packages/web-integration/src/playwright/ai-fixture.ts +++ b/packages/web-integration/src/playwright/ai-fixture.ts @@ -1,3 +1,4 @@ +import { resolveBrowserAgentRuntimeOptions } from '@/common/browser-agent'; import { PlaywrightAgent, PlaywrightBrowserAgent, @@ -155,6 +156,15 @@ export const PlaywrightAiFixture = (options?: PlaywrightAiFixtureOptions) => { ); } + const runtimeOptions = resolveBrowserAgentRuntimeOptions({ + agentName: 'PlaywrightAiFixture', + pageScope: autoFollowNewPage ? 'browser' : 'page', + forceSameTabNavigation: autoFollowNewPage + ? undefined + : forceSameTabNavigation, + autoFollowNewPage, + }); + const commonAgentOpts = { testId: reportTag, reportFileName: reportTag, @@ -169,11 +179,11 @@ export const PlaywrightAiFixture = (options?: PlaywrightAiFixtureOptions) => { const agent = autoFollowNewPage ? new PlaywrightBrowserAgent(page.context(), page, { ...commonAgentOpts, - autoFollowNewPage: true, + autoFollowNewPage: runtimeOptions.autoFollowNewPage, }) : new PlaywrightAgent(page, { ...commonAgentOpts, - forceSameTabNavigation, + forceSameTabNavigation: runtimeOptions.forceSameTabNavigation, }); pageAgentMap[idForPage] = agent; const records = getAgentRecordsForTest(testInfo); diff --git a/packages/web-integration/src/playwright/browser-agent.ts b/packages/web-integration/src/playwright/browser-agent.ts index f571a8803e..15b1a847d3 100644 --- a/packages/web-integration/src/playwright/browser-agent.ts +++ b/packages/web-integration/src/playwright/browser-agent.ts @@ -1,13 +1,14 @@ import { type BrowserAgentAdapter, BrowserAgentPageController, + BrowserAwareAgent, + resolveBrowserAgentRuntimeOptions, } from '@/common/browser-agent'; import { applyForceChromeSelectRendering, isRetryableBrowserNavigationError, } from '@/common/web-agent'; import type { WebPageAgentOpt } from '@/web-element'; -import { Agent as PageAgent } from '@midscene/core/agent'; import { getDebug } from '@midscene/shared/logger'; import type { BrowserContext as PlaywrightBrowserContext, @@ -41,11 +42,17 @@ export type PlaywrightBrowserAgentCreateOpt = PlaywrightBrowserAgentOpt & { initialPage?: PlaywrightPage; }; -export class PlaywrightBrowserAgent extends PageAgent { - private readonly pageController: BrowserAgentPageController< +export class PlaywrightBrowserAgent extends BrowserAwareAgent< + PlaywrightWebPage, + PlaywrightPage, + PlaywrightPage +> { + private get pageController(): BrowserAgentPageController< PlaywrightPage, PlaywrightPage - >; + > { + return this.getPageController(); + } protected isRetryableContextError(error: unknown): boolean { return isRetryableBrowserNavigationError(error); @@ -67,29 +74,32 @@ export class PlaywrightBrowserAgent extends PageAgent { ); } - const { - autoFollowNewPage = false, - newPageTimeout = 5000, - ...agentOpts - } = opts ?? {}; + const { autoFollowNewPage, newPageTimeout, ...agentOpts } = opts ?? {}; + const runtimeOptions = resolveBrowserAgentRuntimeOptions({ + agentName: 'PlaywrightBrowserAgent', + pageScope: 'browser', + forceSameTabNavigation: (opts as WebPageAgentOpt | undefined) + ?.forceSameTabNavigation, + autoFollowNewPage, + newPageTimeout, + }); const { forceChromeSelectRendering } = agentOpts; const webPage = new PlaywrightWebPage(initialPage, { ...agentOpts, - forceSameTabNavigation: false, + forceSameTabNavigation: runtimeOptions.forceSameTabNavigation, }); - super(webPage, agentOpts); - - this.pageController = new BrowserAgentPageController({ + const pageController = new BrowserAgentPageController({ agentName: 'PlaywrightBrowserAgent', adapter: createPlaywrightBrowserAdapter(context), - getActivePage: () => this.interface.underlyingPage as PlaywrightPage, + getActivePage: () => webPage.underlyingPage as PlaywrightPage, setActivePageValue: (page) => { - this.interface.underlyingPage = page; + webPage.underlyingPage = page; }, - autoFollowNewPage, - newPageTimeout, + autoFollowNewPage: runtimeOptions.autoFollowNewPage, + newPageTimeout: runtimeOptions.newPageTimeout, debug, }); + super(webPage, agentOpts, pageController); applyForceChromeSelectRendering( initialPage, @@ -132,7 +142,6 @@ export class PlaywrightBrowserAgent extends PageAgent { } async destroy() { - this.pageController.destroy(); await super.destroy(); } } diff --git a/packages/web-integration/src/playwright/page-agent.ts b/packages/web-integration/src/playwright/page-agent.ts index a096b23033..471e3d7189 100644 --- a/packages/web-integration/src/playwright/page-agent.ts +++ b/packages/web-integration/src/playwright/page-agent.ts @@ -1,9 +1,12 @@ +import { + BrowserAwareAgent, + resolveBrowserAgentRuntimeOptions, +} from '@/common/browser-agent'; import { applyForceChromeSelectRendering, isRetryableBrowserNavigationError, } from '@/common/web-agent'; import type { WebPageAgentOpt } from '@/web-element'; -import { Agent as PageAgent } from '@midscene/core/agent'; import { getDebug } from '@midscene/shared/logger'; import type { Page as PlaywrightPage } from 'playwright'; import { forceClosePopup } from '../puppeteer/base-page'; @@ -11,7 +14,11 @@ import { WebPage as PlaywrightWebPage } from './page'; const debug = getDebug('playwright:agent'); -export class PlaywrightPageAgent extends PageAgent { +export class PlaywrightPageAgent extends BrowserAwareAgent< + PlaywrightWebPage, + PlaywrightPage, + PlaywrightPage +> { protected isRetryableContextError(error: unknown): boolean { return isRetryableBrowserNavigationError(error); } @@ -25,10 +32,14 @@ export class PlaywrightPageAgent extends PageAgent { const webPage = new PlaywrightWebPage(page, opts); super(webPage, opts); - const { forceSameTabNavigation = true, forceChromeSelectRendering } = - opts ?? {}; + const { forceSameTabNavigation, forceChromeSelectRendering } = opts ?? {}; + const runtimeOptions = resolveBrowserAgentRuntimeOptions({ + agentName: 'PlaywrightPageAgent', + pageScope: 'page', + forceSameTabNavigation, + }); - if (forceSameTabNavigation) { + if (runtimeOptions.forceSameTabNavigation) { forceClosePopup(page, debug); } diff --git a/packages/web-integration/src/puppeteer/agent-launcher.ts b/packages/web-integration/src/puppeteer/agent-launcher.ts index 51dd19c58a..f186a91416 100644 --- a/packages/web-integration/src/puppeteer/agent-launcher.ts +++ b/packages/web-integration/src/puppeteer/agent-launcher.ts @@ -3,6 +3,7 @@ import path from 'node:path'; import { getDebug } from '@midscene/shared/logger'; import { assert } from '@midscene/shared/utils'; +import { resolveBrowserAgentRuntimeOptions } from '@/common/browser-agent'; import { defaultViewportHeight, defaultViewportWidth, @@ -385,10 +386,6 @@ export async function puppeteerAgentForTarget( const { aiActionContext, ...preferenceToUse } = preference ?? {}; - const forceSameTabNavigation = - typeof target.forceSameTabNavigation !== 'undefined' - ? target.forceSameTabNavigation - : true; const mode = target.mode ?? 'page'; if (mode !== 'page' && mode !== 'browser') { @@ -397,20 +394,12 @@ export async function puppeteerAgentForTarget( ); } - if (mode === 'page' && target.autoFollowNewPage) { - throw new Error( - '[midscene] autoFollowNewPage requires browser mode. Use browser: or web.mode: browser.', - ); - } - - if ( - mode === 'browser' && - typeof target.forceSameTabNavigation !== 'undefined' - ) { - throw new Error( - '[midscene] forceSameTabNavigation cannot be used in browser mode. Use page: or web.mode: page when same-tab navigation is required.', - ); - } + const runtimeOptions = resolveBrowserAgentRuntimeOptions({ + agentName: 'YAML web target', + pageScope: mode, + forceSameTabNavigation: target.forceSameTabNavigation, + autoFollowNewPage: target.autoFollowNewPage, + }); const commonAgentOpts = { ...preferenceToUse, @@ -427,11 +416,11 @@ export async function puppeteerAgentForTarget( ? await PuppeteerBrowserAgent.create(page.browser(), { ...commonAgentOpts, initialPage: page, - autoFollowNewPage: target.autoFollowNewPage ?? false, + autoFollowNewPage: runtimeOptions.autoFollowNewPage, }) : new PuppeteerAgent(page, { ...commonAgentOpts, - forceSameTabNavigation, + forceSameTabNavigation: runtimeOptions.forceSameTabNavigation, }); freeFn.push({ diff --git a/packages/web-integration/src/puppeteer/browser-agent.ts b/packages/web-integration/src/puppeteer/browser-agent.ts index 0f5e482cbf..745ea62241 100644 --- a/packages/web-integration/src/puppeteer/browser-agent.ts +++ b/packages/web-integration/src/puppeteer/browser-agent.ts @@ -1,13 +1,14 @@ import { type BrowserAgentAdapter, BrowserAgentPageController, + BrowserAwareAgent, + resolveBrowserAgentRuntimeOptions, } from '@/common/browser-agent'; import { applyForceChromeSelectRendering, isRetryableBrowserNavigationError, } from '@/common/web-agent'; import type { WebPageAgentOpt } from '@/web-element'; -import { Agent as PageAgent } from '@midscene/core/agent'; import { getDebug } from '@midscene/shared/logger'; import type { Browser as PuppeteerBrowser, @@ -43,11 +44,17 @@ export type PuppeteerBrowserAgentCreateOpt = PuppeteerBrowserAgentOpt & { initialPage?: PuppeteerPage; }; -export class PuppeteerBrowserAgent extends PageAgent { - private readonly pageController: BrowserAgentPageController< +export class PuppeteerBrowserAgent extends BrowserAwareAgent< + PuppeteerWebPage, + PuppeteerPage, + PuppeteerTarget +> { + private get pageController(): BrowserAgentPageController< PuppeteerPage, PuppeteerTarget - >; + > { + return this.getPageController(); + } protected isRetryableContextError(error: unknown): boolean { return isRetryableBrowserNavigationError(error); @@ -69,29 +76,32 @@ export class PuppeteerBrowserAgent extends PageAgent { ); } - const { - autoFollowNewPage = false, - newPageTimeout = 5000, - ...agentOpts - } = opts ?? {}; + const { autoFollowNewPage, newPageTimeout, ...agentOpts } = opts ?? {}; + const runtimeOptions = resolveBrowserAgentRuntimeOptions({ + agentName: 'PuppeteerBrowserAgent', + pageScope: 'browser', + forceSameTabNavigation: (opts as WebPageAgentOpt | undefined) + ?.forceSameTabNavigation, + autoFollowNewPage, + newPageTimeout, + }); const { forceChromeSelectRendering } = agentOpts; const webPage = new PuppeteerWebPage(initialPage, { ...agentOpts, - forceSameTabNavigation: false, + forceSameTabNavigation: runtimeOptions.forceSameTabNavigation, }); - super(webPage, agentOpts); - - this.pageController = new BrowserAgentPageController({ + const pageController = new BrowserAgentPageController({ agentName: 'PuppeteerBrowserAgent', adapter: createPuppeteerBrowserAdapter(browser), - getActivePage: () => this.interface.underlyingPage as PuppeteerPage, + getActivePage: () => webPage.underlyingPage as PuppeteerPage, setActivePageValue: (page) => { - this.interface.underlyingPage = page; + webPage.underlyingPage = page; }, - autoFollowNewPage, - newPageTimeout, + autoFollowNewPage: runtimeOptions.autoFollowNewPage, + newPageTimeout: runtimeOptions.newPageTimeout, debug, }); + super(webPage, agentOpts, pageController); applyForceChromeSelectRendering( initialPage, @@ -135,7 +145,6 @@ export class PuppeteerBrowserAgent extends PageAgent { } async destroy() { - this.pageController.destroy(); await super.destroy(); } } diff --git a/packages/web-integration/src/puppeteer/page-agent.ts b/packages/web-integration/src/puppeteer/page-agent.ts index 45ebbf10f2..4fd5688c64 100644 --- a/packages/web-integration/src/puppeteer/page-agent.ts +++ b/packages/web-integration/src/puppeteer/page-agent.ts @@ -1,17 +1,27 @@ +import { + BrowserAwareAgent, + resolveBrowserAgentRuntimeOptions, +} from '@/common/browser-agent'; import { applyForceChromeSelectRendering, isRetryableBrowserNavigationError, } from '@/common/web-agent'; import type { WebPageAgentOpt } from '@/web-element'; -import { Agent as PageAgent } from '@midscene/core/agent'; import { getDebug } from '@midscene/shared/logger'; -import type { Page as PuppeteerPage } from 'puppeteer'; +import type { + Page as PuppeteerPage, + Target as PuppeteerTarget, +} from 'puppeteer'; import { forceClosePopup } from './base-page'; import { PuppeteerWebPage } from './page'; const debug = getDebug('puppeteer:agent'); -export class PuppeteerPageAgent extends PageAgent { +export class PuppeteerPageAgent extends BrowserAwareAgent< + PuppeteerWebPage, + PuppeteerPage, + PuppeteerTarget +> { protected isRetryableContextError(error: unknown): boolean { return isRetryableBrowserNavigationError(error); } @@ -25,10 +35,14 @@ export class PuppeteerPageAgent extends PageAgent { const webPage = new PuppeteerWebPage(page, opts); super(webPage, opts); - const { forceSameTabNavigation = true, forceChromeSelectRendering } = - opts ?? {}; + const { forceSameTabNavigation, forceChromeSelectRendering } = opts ?? {}; + const runtimeOptions = resolveBrowserAgentRuntimeOptions({ + agentName: 'PuppeteerPageAgent', + pageScope: 'page', + forceSameTabNavigation, + }); - if (forceSameTabNavigation) { + if (runtimeOptions.forceSameTabNavigation) { forceClosePopup(page, debug); } diff --git a/packages/web-integration/tests/unit-test/browser-agent-page-controller.test.ts b/packages/web-integration/tests/unit-test/browser-agent-page-controller.test.ts index d8412c06a1..d4b82665d9 100644 --- a/packages/web-integration/tests/unit-test/browser-agent-page-controller.test.ts +++ b/packages/web-integration/tests/unit-test/browser-agent-page-controller.test.ts @@ -1,4 +1,7 @@ -import { BrowserAgentPageController } from '@/common/browser-agent'; +import { + BrowserAgentPageController, + resolveBrowserAgentRuntimeOptions, +} from '@/common/browser-agent'; import { describe, expect, it, vi } from 'vitest'; type PageMock = { @@ -120,3 +123,53 @@ describe('BrowserAgentPageController', () => { ); }); }); + +describe('resolveBrowserAgentRuntimeOptions', () => { + it('keeps page mode locked by default', () => { + expect( + resolveBrowserAgentRuntimeOptions({ + agentName: 'TestPageAgent', + pageScope: 'page', + }), + ).toEqual({ + pageScope: 'page', + forceSameTabNavigation: true, + autoFollowNewPage: false, + newPageTimeout: 5000, + }); + }); + + it('keeps browser mode browser-controlled by default', () => { + expect( + resolveBrowserAgentRuntimeOptions({ + agentName: 'TestBrowserAgent', + pageScope: 'browser', + }), + ).toEqual({ + pageScope: 'browser', + forceSameTabNavigation: false, + autoFollowNewPage: false, + newPageTimeout: 5000, + }); + }); + + it('rejects auto-follow in page mode', () => { + expect(() => + resolveBrowserAgentRuntimeOptions({ + agentName: 'TestPageAgent', + pageScope: 'page', + autoFollowNewPage: true, + }), + ).toThrow('autoFollowNewPage requires browser mode'); + }); + + it('rejects same-tab forcing in browser mode', () => { + expect(() => + resolveBrowserAgentRuntimeOptions({ + agentName: 'TestBrowserAgent', + pageScope: 'browser', + forceSameTabNavigation: false, + }), + ).toThrow('forceSameTabNavigation cannot be used in browser mode'); + }); +});