✨ 增强截图和 OPFS 能力：区域截图、saveTo 持久化、bloburl 读取

CodFrm · CodFrm · commit 2a1314e0a34d · 2026-03-18T12:02:00.000+08:00
- screenshot 支持 selector 参数，通过 CDP clip 实现指定元素区域截图
- screenshot 支持 saveTo 参数，将截图二进制直接保存到 OPFS workspace
- screenshot 返回类型升级为 ScreenshotResult（dataUrl + path + size）
- opfs_read 支持 format: "bloburl"，通过 Offscreen 创建 blob URL
- 提取 opfs_helpers.ts 公共模块，供 opfs_tools 和 agent_dom 复用
- GM API（CAT.agent.dom.screenshot / CAT.agent.opfs.read）同步更新
diff --git a/src/app/service/agent/opfs_helpers.ts b/src/app/service/agent/opfs_helpers.ts
@@ -0,0 +1,84 @@
+// OPFS 工作区公共辅助函数
+// 供 opfs_tools、agent_dom 等模块复用
+
+export const WORKSPACE_ROOT = "agents/workspace";
+
+/** Strip leading `/`, reject `..` segments */
+export function sanitizePath(raw: string): string {
+  const stripped = raw.replace(/^\/+/, "");
+  const segments = stripped.split("/").filter(Boolean);
+  for (const seg of segments) {
+    if (seg === "..") {
+      throw new Error(`Invalid path: ".." is not allowed`);
+    }
+  }
+  return segments.join("/");
+}
+
+/** Navigate into nested directories, creating them as needed */
+export async function getDirectory(
+  root: FileSystemDirectoryHandle,
+  path: string,
+  create = false
+): Promise<FileSystemDirectoryHandle> {
+  const segments = path.split("/").filter(Boolean);
+  let dir = root;
+  for (const seg of segments) {
+    dir = await dir.getDirectoryHandle(seg, { create });
+  }
+  return dir;
+}
+
+/** Get the workspace root directory handle */
+export async function getWorkspaceRoot(create = false): Promise<FileSystemDirectoryHandle> {
+  const opfsRoot = await navigator.storage.getDirectory();
+  return getDirectory(opfsRoot, WORKSPACE_ROOT, create);
+}
+
+/** Split a sanitized path into parent directory path and filename */
+export function splitPath(sanitized: string): { dirPath: string; fileName: string } {
+  const lastSlash = sanitized.lastIndexOf("/");
+  if (lastSlash === -1) {
+    return { dirPath: "", fileName: sanitized };
+  }
+  return {
+    dirPath: sanitized.substring(0, lastSlash),
+    fileName: sanitized.substring(lastSlash + 1),
+  };
+}
+
+/** 将 data URL 解码为二进制 Uint8Array */
+export function decodeDataUrl(dataUrl: string): { data: Uint8Array; mimeType: string } {
+  const match = dataUrl.match(/^data:([^;]+);base64,(.+)$/);
+  if (!match) {
+    throw new Error("Invalid data URL format");
+  }
+  const mimeType = match[1];
+  const base64 = match[2];
+  const binaryStr = atob(base64);
+  const bytes = new Uint8Array(binaryStr.length);
+  for (let i = 0; i < binaryStr.length; i++) {
+    bytes[i] = binaryStr.charCodeAt(i);
+  }
+  return { data: bytes, mimeType };
+}
+
+/** 将二进制数据写入 OPFS workspace 指定路径 */
+export async function writeWorkspaceFile(
+  path: string,
+  data: Uint8Array | string
+): Promise<{ path: string; size: number }> {
+  const safePath = sanitizePath(path);
+  if (!safePath) throw new Error("path is required");
+
+  const workspace = await getWorkspaceRoot(true);
+  const { dirPath, fileName } = splitPath(safePath);
+  const dir = dirPath ? await getDirectory(workspace, dirPath, true) : workspace;
+  const fileHandle = await dir.getFileHandle(fileName, { create: true });
+  const writable = await fileHandle.createWritable();
+  await writable.write(data instanceof Uint8Array ? (data.buffer as ArrayBuffer) : data);
+  await writable.close();
+
+  const size = typeof data === "string" ? new Blob([data]).size : data.byteLength;
+  return { path: safePath, size };
+}
diff --git a/src/app/service/agent/tools/opfs_tools.ts b/src/app/service/agent/tools/opfs_tools.ts
@@ -1,51 +1,15 @@
 import type { ToolDefinition } from "@App/app/service/agent/types";
 import type { ToolExecutor } from "@App/app/service/agent/tool_registry";
+import {
+  sanitizePath,
+  getDirectory,
+  getWorkspaceRoot,
+  splitPath,
+  writeWorkspaceFile,
+} from "@App/app/service/agent/opfs_helpers";
 
-const WORKSPACE_ROOT = "agents/workspace";
-
-/** Strip leading `/`, reject `..` segments */
-export function sanitizePath(raw: string): string {
-  const stripped = raw.replace(/^\/+/, "");
-  const segments = stripped.split("/").filter(Boolean);
-  for (const seg of segments) {
-    if (seg === "..") {
-      throw new Error(`Invalid path: ".." is not allowed`);
-    }
-  }
-  return segments.join("/");
-}
-
-/** Navigate into nested directories, creating them as needed */
-async function getDirectory(
-  root: FileSystemDirectoryHandle,
-  path: string,
-  create = false
-): Promise<FileSystemDirectoryHandle> {
-  const segments = path.split("/").filter(Boolean);
-  let dir = root;
-  for (const seg of segments) {
-    dir = await dir.getDirectoryHandle(seg, { create });
-  }
-  return dir;
-}
-
-/** Get the workspace root directory handle */
-async function getWorkspaceRoot(create = false): Promise<FileSystemDirectoryHandle> {
-  const opfsRoot = await navigator.storage.getDirectory();
-  return getDirectory(opfsRoot, WORKSPACE_ROOT, create);
-}
-
-/** Split a sanitized path into parent directory path and filename */
-function splitPath(sanitized: string): { dirPath: string; fileName: string } {
-  const lastSlash = sanitized.lastIndexOf("/");
-  if (lastSlash === -1) {
-    return { dirPath: "", fileName: sanitized };
-  }
-  return {
-    dirPath: sanitized.substring(0, lastSlash),
-    fileName: sanitized.substring(lastSlash + 1),
-  };
-}
+// re-export sanitizePath 供外部使用
+export { sanitizePath };
 
 // ---- Tool Definitions ----
 
@@ -64,11 +28,18 @@ const OPFS_WRITE_DEFINITION: ToolDefinition = {
 
 const OPFS_READ_DEFINITION: ToolDefinition = {
   name: "opfs_read",
-  description: "Read text content from a file in the workspace.",
+  description:
+    "Read a file from the workspace. For text files returns content. For binary files use format='bloburl' to get a blob URL usable in executeScript (ISOLATED world).",
   parameters: {
     type: "object",
     properties: {
       path: { type: "string", description: "File path relative to workspace root" },
+      format: {
+        type: "string",
+        enum: ["text", "bloburl"],
+        description:
+          "Output format: 'text' (default) returns file content as string; 'bloburl' returns a blob:chrome-extension:// URL for binary files (usable in executeScript ISOLATED world)",
+      },
     },
     required: ["path"],
   },
@@ -97,41 +68,76 @@ const OPFS_DELETE_DEFINITION: ToolDefinition = {
   },
 };
 
+// ---- blob URL 创建（通过 Offscreen） ----
+
+// 创建 blob URL 的回调，由外部注入（Offscreen 通道）
+type CreateBlobUrlFn = (data: ArrayBuffer, mimeType: string) => Promise<string>;
+let createBlobUrlFn: CreateBlobUrlFn | null = null;
+
+/** 注入 Offscreen blob URL 创建函数 */
+export function setCreateBlobUrlFn(fn: CreateBlobUrlFn): void {
+  createBlobUrlFn = fn;
+}
+
+/** 根据文件扩展名推断 MIME 类型 */
+function guessMimeType(path: string): string {
+  const ext = path.split(".").pop()?.toLowerCase() || "";
+  const map: Record<string, string> = {
+    jpg: "image/jpeg",
+    jpeg: "image/jpeg",
+    png: "image/png",
+    gif: "image/gif",
+    webp: "image/webp",
+    svg: "image/svg+xml",
+    mp3: "audio/mpeg",
+    wav: "audio/wav",
+    mp4: "video/mp4",
+    pdf: "application/pdf",
+    json: "application/json",
+    txt: "text/plain",
+    html: "text/html",
+    css: "text/css",
+    js: "application/javascript",
+  };
+  return map[ext] || "application/octet-stream";
+}
+
 // ---- Factory ----
 
 export function createOPFSTools(): {
   tools: Array<{ definition: ToolDefinition; executor: ToolExecutor }>;
 } {
   const writeExecutor: ToolExecutor = {
     execute: async (args: Record<string, unknown>) => {
-      const safePath = sanitizePath(args.path as string);
-      const content = args.content as string;
-      if (!safePath) throw new Error("path is required");
-
-      const workspace = await getWorkspaceRoot(true);
-      const { dirPath, fileName } = splitPath(safePath);
-      const dir = dirPath ? await getDirectory(workspace, dirPath, true) : workspace;
-      const fileHandle = await dir.getFileHandle(fileName, { create: true });
-      const writable = await fileHandle.createWritable();
-      await writable.write(content);
-      await writable.close();
-
-      return JSON.stringify({ path: safePath, size: new Blob([content]).size });
+      const result = await writeWorkspaceFile(args.path as string, args.content as string);
+      return JSON.stringify(result);
     },
   };
 
   const readExecutor: ToolExecutor = {
     execute: async (args: Record<string, unknown>) => {
       const safePath = sanitizePath(args.path as string);
       if (!safePath) throw new Error("path is required");
+      const format = (args.format as string) || "text";
 
       const workspace = await getWorkspaceRoot();
       const { dirPath, fileName } = splitPath(safePath);
       const dir = dirPath ? await getDirectory(workspace, dirPath) : workspace;
       const fileHandle = await dir.getFileHandle(fileName);
       const file = await fileHandle.getFile();
-      const content = await file.text();
 
+      if (format === "bloburl") {
+        if (!createBlobUrlFn) {
+          throw new Error("Blob URL creation not available (Offscreen not initialized)");
+        }
+        const arrayBuffer = await file.arrayBuffer();
+        const mimeType = guessMimeType(safePath);
+        const blobUrl = await createBlobUrlFn(arrayBuffer, mimeType);
+        return JSON.stringify({ path: safePath, blobUrl, size: file.size, mimeType });
+      }
+
+      // 默认 text 模式
+      const content = await file.text();
       return JSON.stringify({ path: safePath, content, size: file.size });
     },
   };
@@ -147,8 +153,8 @@ export function createOPFSTools(): {
       const entries: Array<{ name: string; type: "file" | "directory"; size?: number }> = [];
       for await (const [name, handle] of dir as unknown as AsyncIterable<[string, FileSystemHandle]>) {
         if (handle.kind === "file") {
-          const file = await (handle as FileSystemFileHandle).getFile();
-          entries.push({ name, type: "file", size: file.size });
+          const f = await (handle as FileSystemFileHandle).getFile();
+          entries.push({ name, type: "file", size: f.size });
         } else {
           entries.push({ name, type: "directory" });
         }
diff --git a/src/app/service/agent/types.ts b/src/app/service/agent/types.ts
@@ -252,7 +252,7 @@ export type SkillApiRequest =
 // CAT.agent.opfs API 请求
 export type OPFSApiRequest =
   | { action: "write"; path: string; content: string; scriptUuid: string }
-  | { action: "read"; path: string; scriptUuid: string }
+  | { action: "read"; path: string; format?: "text" | "bloburl"; scriptUuid: string }
   | { action: "list"; path?: string; scriptUuid: string }
   | { action: "delete"; path: string; scriptUuid: string };
 
@@ -339,6 +339,13 @@ export type ScreenshotOptions = {
   quality?: number;
   fullPage?: boolean;
   selector?: string; // CSS 选择器，截取指定元素区域
+  saveTo?: string; // OPFS workspace 相对路径，截图后保存二进制
+};
+
+export type ScreenshotResult = {
+  dataUrl: string; // 原始 data URL
+  path?: string; // saveTo 时返回的 OPFS 路径
+  size?: number; // saveTo 时返回的文件大小（字节）
 };
 
 export type NavigateOptions = {
diff --git a/src/app/service/content/gm_api/cat_agent_dom.ts b/src/app/service/content/gm_api/cat_agent_dom.ts
@@ -6,6 +6,7 @@ import type {
   DomApiRequest,
   ReadPageOptions,
   ScreenshotOptions,
+  ScreenshotResult,
   DomActionOptions,
   NavigateOptions,
   ScrollDirection,
@@ -60,7 +61,7 @@ export default class CATAgentDomApi {
   }
 
   @GMContext.API({ follow: "CAT.agent.dom" })
-  public "CAT.agent.dom.screenshot"(options?: ScreenshotOptions): Promise<string> {
+  public "CAT.agent.dom.screenshot"(options?: ScreenshotOptions): Promise<ScreenshotResult> {
     const ctx = this as unknown as GMBaseContext;
     return ctx.sendMessage("CAT_agentDom", [
       { action: "screenshot", options, scriptUuid: ctx.scriptRes?.uuid || "" } as DomApiRequest,
diff --git a/src/app/service/content/gm_api/cat_agent_opfs.ts b/src/app/service/content/gm_api/cat_agent_opfs.ts
@@ -33,11 +33,14 @@ export default class CATAgentOPFSApi {
   }
 
   @GMContext.API({ follow: "CAT.agent.opfs" })
-  public "CAT.agent.opfs.read"(path: string): Promise<{ path: string; content: string; size: number }> {
+  public "CAT.agent.opfs.read"(
+    path: string,
+    format?: "text" | "bloburl"
+  ): Promise<{ path: string; content?: string; blobUrl?: string; size: number; mimeType?: string }> {
     const ctx = this as unknown as GMBaseContext;
     return ctx.sendMessage("CAT_agentOPFS", [
-      { action: "read", path, scriptUuid: ctx.scriptRes?.uuid || "" } as OPFSApiRequest,
-    ]) as Promise<{ path: string; content: string; size: number }>;
+      { action: "read", path, format, scriptUuid: ctx.scriptRes?.uuid || "" } as OPFSApiRequest,
+    ]) as Promise<{ path: string; content?: string; blobUrl?: string; size: number; mimeType?: string }>;
   }
 
   @GMContext.API({ follow: "CAT.agent.opfs" })
diff --git a/src/app/service/service_worker/agent.ts b/src/app/service/service_worker/agent.ts
@@ -57,7 +57,8 @@ import { SearchConfigRepo } from "@App/app/service/agent/tools/search_config";
 import { createTaskTools } from "@App/app/service/agent/tools/task_tools";
 import { createAskUserTool } from "@App/app/service/agent/tools/ask_user";
 import { createSubAgentTool } from "@App/app/service/agent/tools/sub_agent";
-import { createOPFSTools } from "@App/app/service/agent/tools/opfs_tools";
+import { createOPFSTools, setCreateBlobUrlFn } from "@App/app/service/agent/tools/opfs_tools";
+import { createObjectURL } from "@App/app/service/offscreen/client";
 import { createExecuteScriptTool } from "@App/app/service/agent/tools/execute_script";
 import { executeSkillScript } from "@App/app/service/offscreen/client";
 import { createTabTools } from "@App/app/service/agent/tools/tab_tools";
@@ -194,6 +195,11 @@ export class AgentService {
     this.toolRegistry.registerBuiltin(WEB_FETCH_DEFINITION, new WebFetchExecutor(this.sender));
     this.toolRegistry.registerBuiltin(WEB_SEARCH_DEFINITION, new WebSearchExecutor(this.sender, searchConfigRepo));
     // 注册 OPFS 工作区文件工具
+    // 注入 blob URL 创建函数（通过 Offscreen 的 URL.createObjectURL）
+    setCreateBlobUrlFn(async (data: ArrayBuffer, mimeType: string) => {
+      const blob = new Blob([data], { type: mimeType });
+      return (await createObjectURL(this.sender, { blob, persistence: true })) as string;
+    });
     const opfsTools = createOPFSTools();
     for (const t of opfsTools.tools) {
       this.toolRegistry.registerBuiltin(t.definition, t.executor);
@@ -429,7 +435,7 @@ export class AgentService {
       }
       case "read": {
         const executor = toolMap.get("opfs_read")!;
-        return JSON.parse((await executor.execute({ path: request.path })) as string);
+        return JSON.parse((await executor.execute({ path: request.path, format: request.format })) as string);
       }
       case "list": {
         const executor = toolMap.get("opfs_list")!;
diff --git a/src/app/service/service_worker/agent_dom.test.ts b/src/app/service/service_worker/agent_dom.test.ts
@@ -285,7 +285,7 @@ describe("AgentDomService", () => {
 
       const result = await service.screenshot({ tabId: 1 });
 
-      expect(result).toBe("data:image/jpeg;base64,abc123");
+      expect(result.dataUrl).toBe("data:image/jpeg;base64,abc123");
       expect(mockCaptureVisibleTab).toHaveBeenCalledWith(1, { format: "jpeg", quality: 80 });
     });
   });
diff --git a/src/app/service/service_worker/agent_dom.ts b/src/app/service/service_worker/agent_dom.ts
diff --git a/src/app/service/service_worker/agent_dom_cdp.ts b/src/app/service/service_worker/agent_dom_cdp.ts
