Skip to content

Commit 79d99fb

Browse files
Mag1cFallMag1cFall
committed
feat: DOM debug infrastructure + media extraction hardening
Add opt-in DOM snapshotting (DOM_DEBUG=true) across all controllers for diagnosing selector drift against live AI Studio pages. New modules: - src/debug/dom_snapshot.py - page snapshot with base64/script sanitization - src/media/image_utils.py - unified image extraction (data URI / fetch fallback) - docs/dom-debug.md - debugging guide with selector audit results Changes: - Imagen: detect paid-usage dialog, raise PaidApiKeyRequiredError (403) - Imagen: widen image selectors with 3-tier fallback - Nano: prefer direct img.src extraction over download button - Test: support skipped count, auto-skip Imagen on paid-key gate - Test: update model to imagen-4.0-generate-001 - Format: ruff/black normalization on model_management, page_controller Verified: 10-worker cluster e2e 5/5 passed, 1 skipped (paid key)
1 parent d706a11 commit 79d99fb

17 files changed

Lines changed: 2881 additions & 969 deletions

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -256,4 +256,5 @@ docker/.env
256256
docker/my_*.json
257257

258258
# Test output files
259-
test/test_output/
259+
test/test_output/
260+
test/dom_snapshots/

‎_selector_audit.py‎

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
import os
2+
import sys
3+
import re
4+
import glob
5+
from html.parser import HTMLParser
6+
7+
SNAPSHOT_DIR = os.path.join(os.path.dirname(__file__), "test", "dom_snapshots")
8+
9+
class TagCollector(HTMLParser):
10+
def __init__(self):
11+
super().__init__()
12+
self.tags = []
13+
def handle_starttag(self, tag, attrs):
14+
self.tags.append((tag, dict(attrs)))
15+
16+
def load_snapshot(path):
17+
with open(path, encoding="utf-8") as f:
18+
return f.read()
19+
20+
def css_match_count(html_text, selector):
21+
try:
22+
from lxml import html as lxml_html
23+
from lxml.cssselect import CSSSelector, SelectorSyntaxError
24+
sel_clean = selector.strip()
25+
if sel_clean.startswith("//"):
26+
return -1
27+
if ":has-text(" in sel_clean or ":text-is(" in sel_clean:
28+
return -1
29+
if ":has(" in sel_clean:
30+
return -1
31+
try:
32+
doc = lxml_html.fromstring(html_text)
33+
except Exception:
34+
return -2
35+
try:
36+
css = CSSSelector(sel_clean)
37+
return len(css(doc))
38+
except SelectorSyntaxError:
39+
return -1
40+
except ImportError:
41+
return -3
42+
43+
SELECTORS = {
44+
"chat": {
45+
"PROMPT_TEXTAREA[0]": 'ms-prompt-box textarea[aria-label="Enter a prompt"]',
46+
"PROMPT_TEXTAREA[1]": 'ms-prompt-box textarea[placeholder="Start typing a prompt"]',
47+
"PROMPT_TEXTAREA[2]": "ms-prompt-box .prompt-box-container .text-wrapper textarea",
48+
"PROMPT_TEXTAREA[3]": "ms-prompt-box textarea",
49+
"SUBMIT_BUTTON[0]": 'ms-run-button button[type="submit"]',
50+
"INSERT_BUTTON[0]": 'button[data-test-id="add-media-button"]',
51+
"INSERT_BUTTON[2]": "ms-add-media-button button",
52+
"RESPONSE_CONTAINER": "ms-chat-turn .chat-turn-container.model",
53+
"RESPONSE_TEXT": "ms-cmark-node.cmark-node",
54+
"LOADING_SPINNER[0]": 'ms-run-button button[type="submit"] svg .stoppable-spinner',
55+
"ZERO_STATE": "ms-zero-state",
56+
"ADVANCED_SETTINGS": 'button[aria-label="Expand or collapse advanced settings"]',
57+
"MAX_OUTPUT_TOKENS": 'input[aria-label="Maximum output tokens"]',
58+
"STOP_SEQUENCE_INPUT": 'input[aria-label="Add stop token"]',
59+
"SYSTEM_INSTRUCTIONS_BTN": 'button[aria-label="System instructions"]',
60+
"SYSTEM_INSTRUCTIONS_TA": 'textarea[aria-label="System instructions"]',
61+
"MODEL_SELECTOR_CARD_TITLE": ".model-selector-card .title",
62+
"MODEL_SELECTOR_CARD_NAME": '[data-test-id="model-name"]',
63+
"GROUNDING_TOGGLE": 'div[data-test-id="searchAsAToolTooltip"] mat-slide-toggle button',
64+
"THINKING_MODE_TOGGLE": 'mat-slide-toggle[data-test-toggle="enable-thinking"]',
65+
"EDIT_MSG_BUTTON": 'button[aria-label="Edit"].toggle-edit-button',
66+
"URL_CONTEXT_TOGGLE": 'ms-browse-as-a-tool mat-slide-toggle button[role="switch"]',
67+
},
68+
"imagen": {
69+
"ROOT": "ms-image-prompt",
70+
"PROMPT_INPUT[0]": 'ms-prompt-input-wrapper textarea[aria-label="Enter a prompt to generate an image"]',
71+
"PROMPT_INPUT[1]": 'textarea[aria-label="Enter a prompt to generate an image"]',
72+
"RUN_BUTTON[0]": 'ms-run-button button[type="submit"]',
73+
"GALLERY_CONTAINER": "ms-image-generation-gallery",
74+
"GALLERY_ITEM": "ms-image-generation-gallery-image",
75+
"GEN_IMAGE": "ms-image-generation-gallery-image img.loaded-image",
76+
"GEN_IMAGE_ALT1": "ms-image-generation-gallery-image .image-container img",
77+
"GEN_IMAGE_ALT2": "ms-image-generation-gallery img",
78+
"PAID_DIALOG": "ms-paid-usage-dialog",
79+
"PAID_CLOSE": 'ms-paid-usage-dialog button[aria-label="close"]',
80+
"DOWNLOAD_BTN": 'ms-image-generation-gallery-image button[aria-label="Download this image"]',
81+
"SETTINGS_PANEL": "ms-run-settings",
82+
"ASPECT_RATIO_BTN": "ms-run-settings ms-aspect-ratio-radio-button button",
83+
},
84+
"nano": {
85+
"IMAGE_CHUNK": "ms-chat-turn ms-prompt-chunk ms-image-chunk",
86+
"NANO_GEN_IMAGE": "ms-chat-turn ms-image-chunk img.loaded-image",
87+
"DOWNLOAD_BTN": 'ms-chat-turn ms-image-chunk button[aria-label="Download"]',
88+
"SETTINGS_PANEL": "ms-run-settings",
89+
},
90+
"tts": {
91+
"ROOT": "ms-speech-prompt",
92+
"AUDIO_PLAYER": ".speech-prompt-footer audio[controls]",
93+
"RUN_BUTTON[0]": 'ms-run-button button[type="submit"]',
94+
"SINGLE_TEXT_INPUT": 'textarea[placeholder="Start writing or paste text here to generate speech"]',
95+
"MULTI_RAW_INPUT": "textarea.multi-speaker-raw-prompt",
96+
"MODE_SELECTOR": "ms-tts-mode-selector",
97+
"VOICE_DROPDOWN": "ms-voice-selector mat-select",
98+
"SETTINGS_PANEL": "ms-speech-run-settings",
99+
},
100+
}
101+
102+
SNAPSHOT_TYPE_MAP = {
103+
"chat": ["chat_"],
104+
"imagen": ["imagen_"],
105+
"nano": ["nano_"],
106+
"tts": ["tts_"],
107+
}
108+
109+
def pick_snapshots(category):
110+
patterns = SNAPSHOT_TYPE_MAP[category]
111+
files = sorted(glob.glob(os.path.join(SNAPSHOT_DIR, "*.html")))
112+
matched = []
113+
for f in files:
114+
base = os.path.basename(f)
115+
for p in patterns:
116+
parts = base.split("_", 3)
117+
if len(parts) >= 4 and p in parts[3]:
118+
matched.append(f)
119+
break
120+
if len(matched) > 3:
121+
return [matched[0], matched[len(matched)//2], matched[-1]]
122+
return matched
123+
124+
def main():
125+
if not os.path.isdir(SNAPSHOT_DIR):
126+
print(f"❌ 快照目录不存在: {SNAPSHOT_DIR}")
127+
sys.exit(1)
128+
129+
all_files = glob.glob(os.path.join(SNAPSHOT_DIR, "*.html"))
130+
print(f"📁 快照目录: {SNAPSHOT_DIR}")
131+
print(f"📄 快照数量: {len(all_files)}\n")
132+
133+
for category, selectors in SELECTORS.items():
134+
snapshots = pick_snapshots(category)
135+
if not snapshots:
136+
print(f"\n{'='*60}")
137+
print(f"⚠️ {category.upper()}: 无快照可用")
138+
continue
139+
140+
print(f"\n{'='*60}")
141+
print(f"🔍 {category.upper()} (采样 {len(snapshots)} 个快照)")
142+
print(f"{'='*60}")
143+
144+
for snap_path in snapshots:
145+
snap_name = os.path.basename(snap_path)
146+
html = load_snapshot(snap_path)
147+
print(f"\n 📄 {snap_name} ({len(html):,} chars)")
148+
149+
for name, sel in selectors.items():
150+
count = css_match_count(html, sel)
151+
if count == -1:
152+
icon = "⏭️"
153+
detail = "Playwright-only 语法"
154+
elif count == -2:
155+
icon = "⚠️"
156+
detail = "HTML 解析失败"
157+
elif count == -3:
158+
icon = "❌"
159+
detail = "lxml 未安装"
160+
print(f" {icon} lxml 未安装，无法继续")
161+
sys.exit(1)
162+
elif count == 0:
163+
icon = "❌"
164+
detail = "未命中"
165+
else:
166+
icon = "✅"
167+
detail = f"命中 {count} 个"
168+
169+
short_sel = sel if len(sel) <= 60 else sel[:57] + "..."
170+
print(f" {icon} {name:30s} | {detail:20s} | {short_sel}")
171+
172+
print(f"\n{'='*60}")
173+
print("图例: ✅=命中 ❌=未命中 ⏭️=Playwright专用语法(无法静态验证)")
174+
print(f"{'='*60}")
175+
176+
if __name__ == "__main__":
177+
main()

‎docs/dom-debug.md‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
# AIStudio2API DOM 调试指南
2+
3+
## 概述
4+
5+
AIStudio2API 通过 Playwright 自动操控 Google AI Studio 页面。由于 AI Studio 的 DOM 结构会频繁变更，选择器失效是最常见的故障类型。本项目内建了 DOM 快照调试基础设施，可在关键操作点自动抓取页面状态用于诊断。
6+
7+
## 快速开始
8+
9+
### 启用 DOM 调试
10+
11+
```bash
12+
# 环境变量
13+
set DOM_DEBUG=true
14+
15+
# 单 Worker 启动
16+
set PYTHONPATH=src && uv run python src/launch_camoufox.py --headless
17+
18+
# 10 Worker 集群启动（通过 Manager）
19+
uv run python src/app_launcher.py
20+
# 然后 POST http://127.0.0.1:9000/api/control/start
21+
```
22+
23+
### 快照存储
24+
25+
- 路径: `test/dom_snapshots/`
26+
- 命名: `{序号}_{时间}_{阶段}_{请求ID}.html`
27+
- 已被 `.gitignore` 排除
28+
29+
### 快照内容
30+
31+
快照经过清洗处理（`src/debug/dom_snapshot.py`）：
32+
- 移除 `<script>` 标签和 inline 事件
33+
- 截断 base64 `data:` URI（保留前 100 字符）
34+
- 截断超长 SVG `d=` 属性
35+
- 保留完整 DOM 结构和所有属性（class/aria-label/data-test-id 等）
36+
37+
## 快照抓取点
38+
39+
### Chat 流程 (`page_controller.py`)
40+
| 快照名 | 触发时机 |
41+
|---|---|
42+
| `chat_nav_ready` | 导航到新聊天完成 |
43+
| `chat_cleared` | 聊天记录清空后 |
44+
| `chat_model_switch_{model}` | 模型切换完成 |
45+
| `chat_google_search_{on/off}` | Google Search toggle |
46+
| `chat_params_ready` | 参数设置完成 |
47+
| `chat_prompt_filled` | 提示词填入后 |
48+
| `chat_submit` | 提交后 |
49+
| `chat_max_tokens_{n}` | Max tokens 设置 |
50+
| `chat_temperature_{t}` | Temperature 设置 |
51+
| `chat_top_p_{p}` | Top-P 设置 |
52+
53+
### Imagen 流程 (`imagen_controller.py`)
54+
| 快照名 | 触发时机 |
55+
|---|---|
56+
| `imagen_nav` | 导航到 Imagen 页面 |
57+
| `imagen_set_ratio` | 画面比例设置 |
58+
| `imagen_prompt` | 提示词填入 |
59+
| `imagen_run` | 点击生成 |
60+
| `imagen_paid_dialog` | 检测到 paid API key 弹窗 |
61+
| `imagen_paid_dialog_closed` | 弹窗关闭后 |
62+
| `imagen_result` | 生成结果就绪 |
63+
| `imagen_timeout` | 超时 |
64+
65+
### Nano 流程 (`nano_controller.py`)
66+
| 快照名 | 触发时机 |
67+
|---|---|
68+
| `nano_nav` | 导航到新聊天 |
69+
| `nano_prompt` | 提示词填入 |
70+
| `nano_run` | 点击生成 |
71+
| `nano_content` | 内容生成完成 |
72+
| `nano_timeout` | 超时 |
73+
74+
### TTS 流程 (`tts_controller.py`)
75+
| 快照名 | 触发时机 |
76+
|---|---|
77+
| `tts_nav` | 导航到 TTS 页面 |
78+
| `tts_mode_switched_{mode}` | 模式切换 |
79+
| `tts_voice_set_{voice}` | 语音设置 |
80+
| `tts_text_filled` | 文本填入 |
81+
| `tts_multi_filled` | 多说话人脚本填入 |
82+
| `tts_run` | 点击运行 |
83+
| `tts_audio_found` | 音频就绪 |
84+
| `tts_timeout` | 超时 |
85+
86+
## 选择器审计（2026-04-01）
87+
88+
通过 `_selector_audit.py` 脚本对 46 个快照进行自动化验证的结果：
89+
90+
### Chat 选择器 (`src/config/selectors.py`)
91+
92+
| 选择器 | 状态 | 说明 |
93+
|---|---|---|
94+
| `ms-prompt-box textarea[aria-label="Enter a prompt"]` | ✅ 稳定 | 主选择器，3/3 命中 |
95+
| `ms-prompt-box textarea[placeholder="Start typing a prompt"]` | ⚠️ 废弃 | 3/3 未命中，placeholder 已变更 |
96+
| `ms-prompt-box textarea` | ✅ 稳定 | 兜底选择器 |
97+
| `ms-run-button button[type="submit"]` | ✅ 稳定 | Run 按钮 |
98+
| `button[data-test-id="add-media-button"]` | ✅ 稳定 | 文件上传 |
99+
| `ms-zero-state` | ✅ 稳定 | 空聊天状态 |
100+
| `button[aria-label="Expand or collapse advanced settings"]` | ✅ 稳定 | 高级设置 |
101+
| `input[aria-label="Maximum output tokens"]` | ✅ 稳定 | |
102+
| `.model-selector-card .title` | ✅ 稳定 | 模型选择器 |
103+
| `[data-test-id="model-name"]` | ✅ 稳定 | |
104+
| `div[data-test-id="searchAsAToolTooltip"] mat-slide-toggle button` | ✅ 稳定 | Google Search |
105+
| `mat-slide-toggle[data-test-toggle="enable-thinking"]` | ✅ 条件 | 仅在支持 thinking 的模型上出现 |
106+
| `ms-chat-turn .chat-turn-container.model` | ✅ 条件 | 仅有回复时出现 |
107+
| `textarea[aria-label="System instructions"]` | ✅ 条件 | 仅展开系统指令面板时出现 |
108+
109+
### Imagen 选择器 (`src/config/imagen_selectors.py`)
110+
111+
| 选择器 | 状态 | 说明 |
112+
|---|---|---|
113+
| `ms-image-prompt` | ✅ 稳定 | 页面根元素 |
114+
| `ms-prompt-input-wrapper textarea[...]` | ⚠️ 废弃 | 3/3 未命中，`ms-prompt-input-wrapper` 已不存在 |
115+
| `textarea[aria-label="Enter a prompt to generate an image"]` | ✅ 稳定 | 应作为主选择器 |
116+
| `ms-run-button button[type="submit"]` | ✅ 稳定 | |
117+
| `ms-run-settings` | ✅ 稳定 | 设置面板 |
118+
| `ms-run-settings ms-aspect-ratio-radio-button button` | ✅ 稳定 | 5 个按钮 |
119+
| `ms-paid-usage-dialog` | ✅ 条件 | paid key 弹窗存在时命中 |
120+
| `ms-image-generation-gallery-image img.loaded-image` | ✅ 条件 | 生成完成后命中 |
121+
122+
### Nano 选择器 (`src/config/nano_selectors.py`)
123+
124+
| 选择器 | 状态 | 说明 |
125+
|---|---|---|
126+
| `ms-run-settings` | ✅ 稳定 | |
127+
| `ms-chat-turn ms-image-chunk img.loaded-image` | ✅ 条件 | 图片生成后才出现 |
128+
129+
### TTS 选择器 (`src/config/tts_selectors.py`)
130+
131+
| 选择器 | 状态 | 说明 |
132+
|---|---|---|
133+
| `ms-speech-prompt` | ✅ 稳定 | 页面根 |
134+
| `ms-run-button button[type="submit"]` | ✅ 稳定 | |
135+
| `ms-tts-mode-selector` | ✅ 条件 | 设置面板加载后出现 |
136+
| `ms-voice-selector mat-select` | ✅ 条件 | 同上 |
137+
| `.speech-prompt-footer audio[controls]` | ✅ 条件 | 音频生成后出现 |
138+
139+
## 诊断工作流
140+
141+
### 步骤 1：复现故障
142+
```bash
143+
set DOM_DEBUG=true
144+
# 触发失败的请求
145+
```
146+
147+
### 步骤 2：找到相关快照
148+
```bash
149+
# 列出最近的快照
150+
dir /od test\dom_snapshots\*.html
151+
```
152+
153+
### 步骤 3：检查 DOM 结构
154+
155+
在快照 HTML 中搜索：
156+
- 自定义元素标签名（`ms-*`）
157+
- `aria-label` 属性值
158+
- `data-test-id` 属性值
159+
- class 名变化
160+
161+
### 步骤 4：修复选择器
162+
163+
在 `src/config/` 目录下更新对应的选择器文件，优先使用：
164+
1. `data-test-id`（最稳定）
165+
2. `aria-label`（语义稳定）
166+
3. 自定义元素标签 `ms-*`（结构稳定）
167+
4. class 名（最不稳定）
168+
169+
### 步骤 5：验证
170+
```bash
171+
python _selector_audit.py
172+
```
173+
174+
## 环境变量参考
175+
176+
| 变量 | 默认值 | 说明 |
177+
|---|---|---|
178+
| `DOM_DEBUG` | `""` | 设为 `true` 启用 DOM 快照 |
179+
| `HOST_OS_FOR_SHORTCUT` | 自动检测 | 键盘快捷键修饰键（macOS 用 Meta） |
180+
181+
## 注意事项
182+
183+
- 快照文件可能很大（200-400KB），定期清理 `test/dom_snapshots/`
184+
- `DOM_DEBUG` 关闭时，`dump_page` 是零开销空操作
185+
- 快照中的 base64 数据已截断，无法用于恢复原始媒体

‎src/api/routes.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -552,7 +552,11 @@ async def generate_image(
552552
if not page_instance or page_instance.is_closed():
553553
raise HTTPException(status_code=503, detail=f"[{req_id}] 浏览器页面不可用。")
554554

555-
from media import process_image_request, ImageGenerationConfig
555+
from media import (
556+
process_image_request,
557+
ImageGenerationConfig,
558+
PaidApiKeyRequiredError,
559+
)
556560

557561
def check_client_disconnected(stage: str = "") -> bool:
558562
return False
@@ -577,6 +581,9 @@ def check_client_disconnected(stage: str = "") -> bool:
577581
except ClientDisconnectedError as e:
578582
logger.warning(f"[{req_id}] 客户端断开: {e}")
579583
raise HTTPException(status_code=499, detail=str(e))
584+
except PaidApiKeyRequiredError as e:
585+
logger.warning(f"[{req_id}] Imagen 需要付费 API key: {e}")
586+
raise HTTPException(status_code=403, detail=f"[{req_id}] {e}")
580587
except TimeoutError as e:
581588
logger.error(f"[{req_id}] 图片生成超时: {e}")
582589
raise HTTPException(status_code=504, detail=f"[{req_id}] 图片生成超时: {e}")

0 commit comments

Comments
 (0)