Skip to content

Commit b00c148

Browse files
CodFrmCodFrm
committed
♻️ 重构 Agent E2E 测试：两阶段启动 + 共享 fixture + 修复 Repo 缓存 bug
- 提取两阶段启动逻辑到 fixtures.ts (testWithUserScripts) - 提取共享工具函数到 utils.ts (patchScriptCode, autoApprovePermissions, runTestScript, runInlineTestScript) - agent-fixtures 在 Phase 1 写入 mock model 配置，修复 Repo enableCache() 导致的 "No model configured" 问题 - 简化 gm-api.spec.ts，复用共享 fixture 和工具函数
1 parent 5f61397 commit b00c148

7 files changed

Lines changed: 252 additions & 282 deletions

File tree

‎e2e/agent-conversation.spec.ts‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { expect } from "@playwright/test";
2-
import { test, makeTextSSE, makeToolCallSSE, runAgentTestScript } from "./agent-fixtures";
2+
import { test, makeTextSSE, makeToolCallSSE } from "./agent-fixtures";
3+
import { runInlineTestScript } from "./utils";
34

45
const TARGET_URL = "https://content-security-policy.com/";
56

@@ -46,7 +47,7 @@ test.describe("Agent Conversation API", () => {
4647
})();
4748
`;
4849

49-
const { passed, failed, logs } = await runAgentTestScript(context, extensionId, code, TARGET_URL, 60_000);
50+
const { passed, failed, logs } = await runInlineTestScript(context, extensionId, code, TARGET_URL, 60_000);
5051

5152
console.log(`[agent-basic-chat] passed=${passed}, failed=${failed}`);
5253
if (failed !== 0) console.log("[agent-basic-chat] logs:", logs.join("\n"));
@@ -130,7 +131,7 @@ test.describe("Agent Conversation API", () => {
130131
})();
131132
`;
132133

133-
const { passed, failed, logs } = await runAgentTestScript(context, extensionId, code, TARGET_URL, 60_000);
134+
const { passed, failed, logs } = await runInlineTestScript(context, extensionId, code, TARGET_URL, 60_000);
134135

135136
console.log(`[agent-tool-calling] passed=${passed}, failed=${failed}`);
136137
if (failed !== 0) console.log("[agent-tool-calling] logs:", logs.join("\n"));
@@ -189,7 +190,7 @@ test.describe("Agent Conversation API", () => {
189190
})();
190191
`;
191192

192-
const { passed, failed, logs } = await runAgentTestScript(context, extensionId, code, TARGET_URL, 60_000);
193+
const { passed, failed, logs } = await runInlineTestScript(context, extensionId, code, TARGET_URL, 60_000);
193194

194195
console.log(`[agent-multi-turn] passed=${passed}, failed=${failed}`);
195196
if (failed !== 0) console.log("[agent-multi-turn] logs:", logs.join("\n"));

‎e2e/agent-error-handling.spec.ts‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { expect } from "@playwright/test";
2-
import { test, makeTextSSE, runAgentTestScript } from "./agent-fixtures";
2+
import { test, makeTextSSE } from "./agent-fixtures";
3+
import { runInlineTestScript } from "./utils";
34

45
const TARGET_URL = "https://content-security-policy.com/";
56

@@ -78,7 +79,7 @@ test.describe("Agent Error Handling", () => {
7879
})();
7980
`;
8081

81-
const { passed, failed, logs } = await runAgentTestScript(context, extensionId, code, TARGET_URL, 90_000);
82+
const { passed, failed, logs } = await runInlineTestScript(context, extensionId, code, TARGET_URL, 90_000);
8283

8384
console.log(`[error-retry] passed=${passed}, failed=${failed}`);
8485
if (failed !== 0) console.log("[error-retry] logs:", logs.join("\n"));
@@ -137,7 +138,7 @@ test.describe("Agent Error Handling", () => {
137138
})();
138139
`;
139140

140-
const { passed, failed, logs } = await runAgentTestScript(context, extensionId, code, TARGET_URL, 60_000);
141+
const { passed, failed, logs } = await runInlineTestScript(context, extensionId, code, TARGET_URL, 60_000);
141142

142143
console.log(`[auth-error] passed=${passed}, failed=${failed}`);
143144
if (failed !== 0) console.log("[auth-error] logs:", logs.join("\n"));
@@ -191,7 +192,7 @@ test.describe("Agent Error Handling", () => {
191192
})();
192193
`;
193194

194-
const { passed, failed, logs } = await runAgentTestScript(context, extensionId, code, TARGET_URL, 60_000);
195+
const { passed, failed, logs } = await runInlineTestScript(context, extensionId, code, TARGET_URL, 60_000);
195196

196197
console.log(`[abort] passed=${passed}, failed=${failed}`);
197198
if (failed !== 0) console.log("[abort] logs:", logs.join("\n"));

‎e2e/agent-fixtures.ts‎

Lines changed: 63 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,23 @@
1+
import fs from "fs";
2+
import os from "os";
13
import path from "path";
24
import { test as base, chromium, type BrowserContext, type Route } from "@playwright/test";
3-
import { installScriptByCode } from "./utils";
5+
6+
const pathToExtension = path.resolve(__dirname, "../dist/ext");
7+
const chromeArgs = [`--disable-extensions-except=${pathToExtension}`, `--load-extension=${pathToExtension}`];
8+
9+
function getProxyOptions() {
10+
const proxy =
11+
process.env.E2E_PROXY ||
12+
process.env.https_proxy ||
13+
process.env.http_proxy ||
14+
process.env.HTTPS_PROXY ||
15+
process.env.HTTP_PROXY;
16+
return proxy ? { proxy: { server: proxy } } : {};
17+
}
418

519
/** OpenAI-compatible SSE response for plain text replies */
6-
function makeTextSSE(content: string): string {
20+
export function makeTextSSE(content: string): string {
721
const lines = [
822
`data: ${JSON.stringify({ choices: [{ delta: { role: "assistant", content }, index: 0 }] })}`,
923
`data: ${JSON.stringify({ choices: [{ delta: {}, index: 0, finish_reason: "stop" }], usage: { prompt_tokens: 10, completion_tokens: 5 } })}`,
@@ -14,10 +28,9 @@ function makeTextSSE(content: string): string {
1428
}
1529

1630
/** OpenAI-compatible SSE response for tool_calls */
17-
function makeToolCallSSE(toolCalls: Array<{ id: string; name: string; arguments: string }>): string {
31+
export function makeToolCallSSE(toolCalls: Array<{ id: string; name: string; arguments: string }>): string {
1832
const lines: string[] = [];
1933
for (const tc of toolCalls) {
20-
// First chunk: tool call start with name
2134
lines.push(
2235
`data: ${JSON.stringify({
2336
choices: [
@@ -38,7 +51,6 @@ function makeToolCallSSE(toolCalls: Array<{ id: string; name: string; arguments:
3851
],
3952
})}`
4053
);
41-
// Second chunk: arguments
4254
lines.push(
4355
`data: ${JSON.stringify({
4456
choices: [
@@ -52,7 +64,6 @@ function makeToolCallSSE(toolCalls: Array<{ id: string; name: string; arguments:
5264
})}`
5365
);
5466
}
55-
// Finish with tool_calls reason
5667
lines.push(
5768
`data: ${JSON.stringify({
5869
choices: [{ delta: {}, index: 0, finish_reason: "tool_calls" }],
@@ -72,62 +83,50 @@ export type AgentFixtures = {
7283
mockLLMResponse: (handler: MockLLMHandler) => void;
7384
};
7485

75-
export { makeTextSSE, makeToolCallSSE };
76-
7786
/**
78-
* Agent test fixtures — 单 context 方案
87+
* Agent test fixtures — 两阶段启动 + mock LLM
7988
*
80-
* 与旧版两阶段方案（启动→关闭→重启）不同，这里在同一个 context 内完成
81-
* userScripts 启用和 model 配置写入，避免了 CI 上 profile 持久化不可靠的问题。
89+
* Phase 1: 启动 → 启用 userScripts → 写入 mock model 配置 → 关闭
90+
* Phase 2: 重启（权限和配置已持久化到 userDataDir）
91+
*
92+
* 必须在 Phase 1 写入 model 配置，因为 Repo 层使用 enableCache()，
93+
* Phase 2 的 SW 启动时会一次性加载 storage 到内存缓存。
94+
* 如果在 Phase 2 SW 启动后才通过 evaluate 写入 storage，
95+
* 内存缓存不会更新，导致 "No model configured" 错误。
8296
*/
8397
export const test = base.extend<AgentFixtures>({
8498
// eslint-disable-next-line no-empty-pattern
8599
context: async ({}, use) => {
86-
const pathToExtension = path.resolve(__dirname, "../dist/ext");
87-
const context = await chromium.launchPersistentContext("", {
100+
const userDataDir = fs.mkdtempSync(path.join(os.tmpdir(), "pw-ext-"));
101+
102+
// Phase 1: 启用 userScripts + 写入 mock model 配置
103+
const ctx1 = await chromium.launchPersistentContext(userDataDir, {
88104
headless: false,
89-
args: ["--headless=new", `--disable-extensions-except=${pathToExtension}`, `--load-extension=${pathToExtension}`],
90-
...(() => {
91-
const proxy =
92-
process.env.E2E_PROXY ||
93-
process.env.https_proxy ||
94-
process.env.http_proxy ||
95-
process.env.HTTPS_PROXY ||
96-
process.env.HTTP_PROXY;
97-
return proxy ? { proxy: { server: proxy } } : {};
98-
})(),
105+
args: ["--headless=new", ...chromeArgs],
99106
});
100-
101-
await use(context);
102-
await context.close();
103-
},
104-
105-
extensionId: async ({ context }, use) => {
106-
// 等待 service worker 启动
107-
let [bg] = context.serviceWorkers();
108-
if (!bg) bg = await context.waitForEvent("serviceworker");
107+
let [bg] = ctx1.serviceWorkers();
108+
if (!bg) bg = await ctx1.waitForEvent("serviceworker", { timeout: 30_000 });
109109
const extensionId = bg.url().split("/")[2];
110110

111-
// 在同一 context 内启用 userScripts 权限
112-
const setupPage = await context.newPage();
113-
await setupPage.goto("chrome://extensions/");
114-
await setupPage.waitForLoadState("domcontentloaded");
115-
await setupPage.waitForTimeout(1_000);
116-
await setupPage.evaluate(async (id) => {
111+
// 启用 userScripts 权限
112+
const extPage = await ctx1.newPage();
113+
await extPage.goto("chrome://extensions/");
114+
await extPage.waitForLoadState("domcontentloaded");
115+
await extPage.waitForFunction(() => !!(chrome as any).developerPrivate, { timeout: 10_000 });
116+
await extPage.evaluate(async (id) => {
117117
await (chrome as any).developerPrivate.updateExtensionConfiguration({
118118
extensionId: id,
119119
userScriptsAccess: true,
120120
});
121121
}, extensionId);
122-
await setupPage.close();
122+
await extPage.close();
123123

124-
// 启用 userScripts 后 SW 可能会重启，重新获取
125-
let currentBg = context.serviceWorkers().find((w) => w.url().includes(extensionId));
124+
// 写入 mock model 配置到 storage（Phase 1 写入，Phase 2 SW 启动时会加载到缓存）
125+
// userScripts 启用后 SW 可能重启，重新获取
126+
let currentBg = ctx1.serviceWorkers().find((w) => w.url().includes(extensionId));
126127
if (!currentBg) {
127-
currentBg = await context.waitForEvent("serviceworker", { timeout: 15_000 });
128+
currentBg = await ctx1.waitForEvent("serviceworker", { timeout: 15_000 });
128129
}
129-
130-
// 写入 mock model 配置到 storage
131130
await currentBg.evaluate(() => {
132131
const modelConfig = {
133132
id: "mock-model",
@@ -148,6 +147,26 @@ export const test = base.extend<AgentFixtures>({
148147
});
149148
});
150149

150+
await ctx1.close();
151+
152+
// Phase 2: 重启，userScripts 权限和 model 配置已持久化
153+
const context = await chromium.launchPersistentContext(userDataDir, {
154+
headless: false,
155+
args: ["--headless=new", ...chromeArgs],
156+
...getProxyOptions(),
157+
});
158+
const [sw] = context.serviceWorkers();
159+
if (!sw) await context.waitForEvent("serviceworker", { timeout: 30_000 });
160+
await use(context);
161+
await context.close();
162+
fs.rmSync(userDataDir, { recursive: true, force: true });
163+
},
164+
165+
extensionId: async ({ context }, use) => {
166+
let [background] = context.serviceWorkers();
167+
if (!background) background = await context.waitForEvent("serviceworker");
168+
const extensionId = background.url().split("/")[2];
169+
151170
// 关闭首次使用引导
152171
const initPage = await context.newPage();
153172
await initPage.goto(`chrome-extension://${extensionId}/src/options.html`, {
@@ -163,7 +182,6 @@ export const test = base.extend<AgentFixtures>({
163182
mockLLMResponse: async ({ context }, use) => {
164183
let currentHandler: MockLLMHandler = () => makeTextSSE("default mock response");
165184

166-
// Set up route interception for mock LLM
167185
await context.route("**/mock-llm.test/**", async (route: Route) => {
168186
const request = route.request();
169187
if (request.method() !== "POST") {
@@ -201,66 +219,3 @@ export const test = base.extend<AgentFixtures>({
201219
await use(setHandler);
202220
},
203221
});
204-
205-
/**
206-
* Auto-approve permission confirm dialogs.
207-
*/
208-
export function autoApprovePermissions(context: BrowserContext): void {
209-
context.on("page", async (page) => {
210-
const url = page.url();
211-
212-
// Auto-approve permission confirm dialogs
213-
if (url.includes("confirm.html")) {
214-
try {
215-
await page.waitForLoadState("domcontentloaded");
216-
const successButtons = page.locator("button.arco-btn-status-success");
217-
await successButtons.first().waitFor({ timeout: 5_000 });
218-
const count = await successButtons.count();
219-
if (count >= 3) {
220-
await successButtons.nth(2).click();
221-
} else {
222-
await successButtons.last().click();
223-
}
224-
console.log("[autoApprove] Permission approved on confirm page");
225-
} catch (e) {
226-
console.log("[autoApprove] Failed to approve:", e);
227-
}
228-
return;
229-
}
230-
});
231-
}
232-
233-
/** Run an agent test script and collect console results */
234-
export async function runAgentTestScript(
235-
context: BrowserContext,
236-
extensionId: string,
237-
code: string,
238-
targetUrl: string,
239-
timeoutMs: number
240-
): Promise<{ passed: number; failed: number; logs: string[] }> {
241-
await installScriptByCode(context, extensionId, code);
242-
autoApprovePermissions(context);
243-
244-
const page = await context.newPage();
245-
const logs: string[] = [];
246-
page.on("console", (msg) => logs.push(msg.text()));
247-
248-
await page.goto(targetUrl, { waitUntil: "domcontentloaded" });
249-
250-
const deadline = Date.now() + timeoutMs;
251-
let passed = -1;
252-
let failed = -1;
253-
while (Date.now() < deadline) {
254-
for (const log of logs) {
255-
const passMatch = log.match(/通过[:：]\s*(\d+)/);
256-
const failMatch = log.match(/失败[:：]\s*(\d+)/);
257-
if (passMatch) passed = parseInt(passMatch[1], 10);
258-
if (failMatch) failed = parseInt(failMatch[1], 10);
259-
}
260-
if (passed >= 0 && failed >= 0) break;
261-
await page.waitForTimeout(500);
262-
}
263-
264-
await page.close();
265-
return { passed, failed, logs };
266-
}

‎e2e/agent-skill.spec.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { expect } from "@playwright/test";
2-
import { test, makeTextSSE, makeToolCallSSE, runAgentTestScript } from "./agent-fixtures";
2+
import { test, makeTextSSE, makeToolCallSSE } from "./agent-fixtures";
3+
import { runInlineTestScript } from "./utils";
34

45
const TARGET_URL = "https://content-security-policy.com/";
56

@@ -121,7 +122,7 @@ test.describe("Agent Skill System", () => {
121122
})();
122123
`;
123124

124-
const { passed, failed, logs } = await runAgentTestScript(context, extensionId, code, TARGET_URL, 90_000);
125+
const { passed, failed, logs } = await runInlineTestScript(context, extensionId, code, TARGET_URL, 90_000);
125126

126127
console.log(`[skill-integration] passed=${passed}, failed=${failed}`);
127128
if (failed !== 0) console.log("[skill-integration] logs:", logs.join("\n"));

0 commit comments

Comments
 (0)