feat(cli): 支持多模态消息内容及图片URL数组

clark-fc · clark-fc · commit 2ec2f34763cc · 2026-06-29T13:36:32.000+08:00
- 扩展聊天消息内容类型，支持文本和图片URL的数组形式
- 处理 --image 参数，将图片URL作为多模态内容附加到最后一条用户消息
- 若无用户消息且指定图片URL，自动创建空用户消息以承载图片内容
- 禁止同时使用内嵌图片内容和 --image 参数，避免冲突
- 将知识搜索接口请求的图片参数字段 image_list 重命名为 images
- 单元测试覆盖多模态内容及图片数组行为验证
- 优化消息解析，支持JSON结构化消息和 role:content 格式
- 更新API类型声明，明确多模态消息结构与字段类型
diff --git a/packages/cli/tests/e2e/knowledge-chat.e2e.test.ts b/packages/cli/tests/e2e/knowledge-chat.e2e.test.ts
@@ -2,16 +2,21 @@ import { tmpdir } from "os";
 import { describe, expect, test } from "vite-plus/test";
 import { parseStdoutJson, runCli } from "./helpers.ts";
 
+interface ContentPart {
+  type: string;
+  text?: string;
+  image_url?: { url: string };
+}
+
 interface DryRunBody {
   endpoint?: string;
   request?: {
     input?: {
-      messages?: Array<{ role: string; content: string }>;
+      messages?: Array<{ role: string; content: string | ContentPart[] }>;
     };
     parameters?: {
       agent_options?: {
         agent_id?: string;
-        image_list?: string[];
       };
     };
     stream?: boolean;
@@ -135,7 +140,7 @@ describe("e2e: knowledge chat", () => {
     expect(msgs[2]?.content).toBe("它怎么工作");
   });
 
-  test("--dry-run + --image 输出 image_list", async () => {
+  test("--dry-run + --image 输出多模态 content 数组", async () => {
     const { stdout, stderr, exitCode } = await runCli(
       [
         "knowledge",
@@ -157,8 +162,50 @@ describe("e2e: knowledge chat", () => {
     );
     expect(exitCode, stderr).toBe(0);
     const data = parseStdoutJson<DryRunBody>(stdout);
-    expect(data.request?.parameters?.agent_options?.image_list).toEqual([
-      "https://example.com/img.jpg",
-    ]);
+    const lastMsg = data.request?.input?.messages?.[0];
+    expect(lastMsg?.role).toBe("user");
+    expect(Array.isArray(lastMsg?.content)).toBe(true);
+    const parts = lastMsg?.content as ContentPart[];
+    expect(parts[0]).toEqual({ type: "text", text: "描述这张图" });
+    expect(parts[1]).toEqual({
+      type: "image_url",
+      image_url: { url: "https://example.com/img.jpg" },
+    });
+  });
+
+  test("--dry-run + --image 无 --message 自动创建空 user message", async () => {
+    const { stdout, stderr, exitCode } = await runCli(
+      [
+        "knowledge",
+        "chat",
+        "--dry-run",
+        "--agent-id",
+        "aid_test",
+        "--workspace-id",
+        "ws_test",
+        "--image",
+        "https://example.com/a.png",
+        "--image",
+        "https://example.com/b.png",
+        "--non-interactive",
+        "--output",
+        "json",
+      ],
+      { DASHSCOPE_API_KEY: "sk-fake-for-dryrun" },
+    );
+    expect(exitCode, stderr).toBe(0);
+    const data = parseStdoutJson<DryRunBody>(stdout);
+    const lastMsg = data.request?.input?.messages?.[0];
+    expect(lastMsg?.role).toBe("user");
+    const parts = lastMsg?.content as ContentPart[];
+    expect(parts[0]).toEqual({ type: "text", text: "" });
+    expect(parts[1]).toEqual({
+      type: "image_url",
+      image_url: { url: "https://example.com/a.png" },
+    });
+    expect(parts[2]).toEqual({
+      type: "image_url",
+      image_url: { url: "https://example.com/b.png" },
+    });
   });
 });
diff --git a/packages/cli/tests/e2e/knowledge-search.e2e.test.ts b/packages/cli/tests/e2e/knowledge-search.e2e.test.ts
@@ -7,7 +7,7 @@ interface DryRunBody {
   request?: {
     query?: string;
     agent_id?: string;
-    image_list?: string[];
+    images?: string[];
     query_history?: Array<{ role: string; content: string }>;
   };
 }
@@ -96,7 +96,7 @@ describe("e2e: knowledge search", () => {
     expect(data.request?.agent_id).toBe("aid_test");
   });
 
-  test("--dry-run + --image 输出 image_list", async () => {
+  test("--dry-run + --image 输出 images", async () => {
     const { stdout, stderr, exitCode } = await runCli(
       [
         "knowledge",
@@ -120,7 +120,7 @@ describe("e2e: knowledge search", () => {
     );
     expect(exitCode, stderr).toBe(0);
     const data = parseStdoutJson<DryRunBody>(stdout);
-    expect(data.request?.image_list).toEqual([
+    expect(data.request?.images).toEqual([
       "https://example.com/a.jpg",
       "https://example.com/b.jpg",
     ]);
diff --git a/packages/commands/src/commands/knowledge/chat.ts b/packages/commands/src/commands/knowledge/chat.ts
@@ -9,22 +9,40 @@ import {
   isInteractive,
   type Config,
   type GlobalFlags,
+  type KnowledgeChatContentPart,
+  type KnowledgeChatMessage,
   type KnowledgeChatRequest,
   type KnowledgeChatStreamChunk,
 } from "bailian-cli-core";
 import { failIfMissing, cmdUsage, emitResult, emitBare, promptText } from "bailian-cli-runtime";
 
-interface ParsedMessage {
-  role: "user" | "assistant";
-  content: string;
-}
-
-function parseMessages(flags: GlobalFlags): ParsedMessage[] {
-  const messages: ParsedMessage[] = [];
+/**
+ * Parse --message flags into KnowledgeChatMessage[].
+ * Supports:
+ *   1. Simple text: "hello" → {role:"user", content:"hello"}
+ *   2. Role prefix: "user:hello" / "assistant:hi" → {role, content}
+ *   3. JSON object: '{"role":"user","content":[...]}' → structured message (advanced)
+ */
+function parseMessages(flags: GlobalFlags): KnowledgeChatMessage[] {
+  const messages: KnowledgeChatMessage[] = [];
   if (flags.message) {
     const validRoles = new Set(["user", "assistant"]);
     const msgs = flags.message as string[];
     for (const m of msgs) {
+      // Try JSON object first (advanced usage)
+      if (m.startsWith("{")) {
+        try {
+          const parsed = JSON.parse(m) as { role?: string; content?: unknown };
+          if (parsed.role && validRoles.has(parsed.role) && parsed.content !== undefined) {
+            messages.push(parsed as KnowledgeChatMessage);
+            continue;
+          }
+        } catch {
+          // Not valid JSON, fall through to simple parsing
+        }
+      }
+
+      // Simple role:content or plain text
       const colonIdx = m.indexOf(":");
       const maybeRole = colonIdx !== -1 ? m.slice(0, colonIdx) : "";
 
@@ -38,6 +56,55 @@ function parseMessages(flags: GlobalFlags): ParsedMessage[] {
   return messages;
 }
 
+/** Check if any message content already contains image_url parts */
+function hasEmbeddedImages(messages: KnowledgeChatMessage[]): boolean {
+  for (const msg of messages) {
+    if (Array.isArray(msg.content)) {
+      if (msg.content.some((p) => p.type === "image_url")) return true;
+    }
+  }
+  return false;
+}
+
+/** Attach --image URLs to the last user message's content (as multimodal array) */
+function attachImagesToLastUserMessage(
+  messages: KnowledgeChatMessage[],
+  imageUrls: string[],
+): void {
+  // Find last user message index
+  let lastUserIdx = -1;
+  for (let i = messages.length - 1; i >= 0; i--) {
+    if (messages[i]!.role === "user") {
+      lastUserIdx = i;
+      break;
+    }
+  }
+
+  // If no user message exists, append an empty one
+  if (lastUserIdx === -1) {
+    messages.push({ role: "user", content: "" });
+    lastUserIdx = messages.length - 1;
+  }
+
+  const target = messages[lastUserIdx]!;
+  const contentParts: KnowledgeChatContentPart[] = [];
+
+  // Preserve existing text content (always include a text part, even if empty)
+  if (typeof target.content === "string") {
+    contentParts.push({ type: "text", text: target.content });
+  } else {
+    // Already an array, extend it
+    contentParts.push(...target.content);
+  }
+
+  // Append image parts
+  for (const url of imageUrls) {
+    contentParts.push({ type: "image_url", image_url: { url } });
+  }
+
+  target.content = contentParts;
+}
+
 /** SSE step_change → human-friendly progress label (TTY only) */
 const STEP_LABELS: Record<string, string> = {
   tool_calling: "🔍 Retrieving...",
@@ -67,7 +134,8 @@ export default defineCommand({
     },
     {
       flag: "--image <url>",
-      description: "Image URL(s) (repeatable)",
+      description:
+        "Image URL (repeatable). Attached to the last user message as multimodal content",
       type: "array",
     },
   ],
@@ -80,12 +148,19 @@ export default defineCommand({
   exampleArgs: [
     '--message "What is RAG?" --agent-id aid-xxx --workspace-id ws-xxx',
     '--message "user:What is RAG?" --message "assistant:RAG is..." --message "How does it work?" --agent-id aid-xxx --workspace-id ws-xxx',
+    '--message "Describe these images" --image https://example.com/a.png --image https://example.com/b.png --agent-id aid-xxx --workspace-id ws-xxx',
   ],
   async run(config: Config, flags: GlobalFlags) {
     let messages = parseMessages(flags);
 
+    const imageUrls = flags.image as string[] | undefined;
+    const hasImages = imageUrls && imageUrls.length > 0;
+
     if (messages.length === 0) {
-      if (isInteractive({ nonInteractive: config.nonInteractive })) {
+      if (hasImages) {
+        // --image without --message: create an empty user message to hold images
+        messages = [{ role: "user", content: "" }];
+      } else if (isInteractive({ nonInteractive: config.nonInteractive })) {
         const hint = await promptText({ message: "Enter your message:" });
         if (!hint) {
           process.stderr.write("Chat cancelled.\n");
@@ -113,6 +188,17 @@ export default defineCommand({
     // API only supports SSE; streamOutput controls whether to print tokens in real-time
     const streamOutput = format === "text" && !!process.stdout.isTTY;
 
+    // Attach --image URLs to messages (multimodal content array)
+    if (hasImages) {
+      if (hasEmbeddedImages(messages)) {
+        throw new BailianError(
+          "Cannot use --image when messages already contain embedded image_url content parts. Use one approach or the other.",
+          ExitCode.USAGE,
+        );
+      }
+      attachImagesToLastUserMessage(messages, imageUrls!);
+    }
+
     const body: KnowledgeChatRequest = {
       input: {
         messages,
@@ -125,11 +211,6 @@ export default defineCommand({
       stream: true,
     };
 
-    const imageUrls = flags.image as string[] | undefined;
-    if (imageUrls && imageUrls.length > 0) {
-      body.parameters.agent_options.image_list = imageUrls;
-    }
-
     const url = knowledgeChatEndpoint(workspaceId);
 
     if (config.dryRun) {
diff --git a/packages/commands/src/commands/knowledge/search.ts b/packages/commands/src/commands/knowledge/search.ts
@@ -89,7 +89,7 @@ export default defineCommand({
 
     const imageUrls = flags.image as string[] | undefined;
     if (imageUrls && imageUrls.length > 0) {
-      body.image_list = imageUrls;
+      body.images = imageUrls;
     }
 
     // Parse query_history JSON for multi-turn context
diff --git a/packages/core/src/types/api.ts b/packages/core/src/types/api.ts
@@ -422,7 +422,7 @@ export interface DashScopeKnowledgeRetrieveResponse {
 export interface KnowledgeSearchRequest {
   query: string;
   agent_id: string;
-  image_list?: string[];
+  images?: string[];
   query_history?: Array<{ role: "user" | "assistant"; content: string }>;
 }
 
@@ -456,15 +456,22 @@ export interface KnowledgeSearchResponse {
 
 // ---- Knowledge Chat (新版 RAG 问答 SSE API, agent_id-based) ----
 
+export type KnowledgeChatContentPart =
+  | { type: "text"; text: string }
+  | { type: "image_url"; image_url: { url: string } };
+
+export interface KnowledgeChatMessage {
+  role: "user" | "assistant";
+  content: string | KnowledgeChatContentPart[];
+}
+
 export interface KnowledgeChatRequest {
   input: {
-    messages: Array<{ role: "user" | "assistant"; content: string }>;
-    request_id?: string;
+    messages: KnowledgeChatMessage[];
   };
   parameters: {
     agent_options: {
       agent_id: string;
-      image_list?: string[];
       user?: {
         user_id?: string;
         workspace_id?: string;
diff --git a/packages/core/src/types/index.ts b/packages/core/src/types/index.ts
@@ -23,6 +23,8 @@ export type {
   DashScopeVideoEditRequest,
   DashScopeVideoRefRequest,
   DashScopeVideoRequest,
+  KnowledgeChatContentPart,
+  KnowledgeChatMessage,
   KnowledgeChatRequest,
   KnowledgeChatStreamChunk,
   KnowledgeRetrieveRequest,
diff --git a/skills/bailian-cli/reference/knowledge.md b/skills/bailian-cli/reference/knowledge.md
@@ -30,7 +30,7 @@ Index: [index.md](index.md)
 | `--message <text>`    | array  | yes      | Message text (repeatable). Supports role:content prefix to set role (e.g. user:hello), defaults to user. Follows OpenAI message format |
 | `--agent-id <id>`     | string | yes      | Q&A service ID (find in console knowledge Q&A page)                                                                                    |
 | `--workspace-id <id>` | string | no       | Workspace ID for API endpoint URL (or set BAILIAN_WORKSPACE_ID)                                                                        |
-| `--image <url>`       | array  | no       | Image URL(s) (repeatable)                                                                                                              |
+| `--image <url>`       | array  | no       | Image URL (repeatable). Attached to the last user message as multimodal content                                                        |
 
 #### Notes
 
@@ -49,6 +49,10 @@ bl knowledge chat --message "What is RAG?" --agent-id aid-xxx --workspace-id ws-
 bl knowledge chat --message "user:What is RAG?" --message "assistant:RAG is..." --message "How does it work?" --agent-id aid-xxx --workspace-id ws-xxx
 ```
 
+```bash
+bl knowledge chat --message "Describe these images" --image https://example.com/a.png --image https://example.com/b.png --agent-id aid-xxx --workspace-id ws-xxx
+```
+
 ### `bl knowledge retrieve`
 
 | Field           | Value                                                                     |

Original file line number	Diff line number	Diff line change
`@@ -89,7 +89,7 @@ export default defineCommand({`
`89`	`89`
`90`	`90`	`const imageUrls = flags.image as string[] \| undefined;`
`91`	`91`	`if (imageUrls && imageUrls.length > 0) {`
`92`		`- body.image_list = imageUrls;`
	`92`	`+ body.images = imageUrls;`
`93`	`93`	`}`
`94`	`94`
`95`	`95`	`// Parse query_history JSON for multi-turn context`