diff --git a/README.md b/README.md index 0032d9dc..89e86efd 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,9 @@ The architecture is designed for simple deployment, efficient static delivery, a | Area | Capabilities | | --- | --- | | Conversations | Multi-branch chat, streaming, retries, edits, feedback, sharing, cloned shared conversations, rich markdown rendering, file cards, model metadata, usage details, and execution traces. | -| Media generation | Dedicated image generation and image edit flow with task-aware routing, OpenAI Images-compatible protocols, generated file storage, preview, download, and run history separated from text chat. | +| Media generation | Dedicated image generation and image edit flow with task-aware routing, provider-native OpenAI, Google, and xAI image protocols, generated file storage, preview, download, and run history separated from text chat. | | Model control plane | Platform model catalog, upstream channels, real upstream models, route bindings, priority and weight routing, model capability JSON, display ordering, vendor mapping, automatic icons, and circuit breaker state. | -| Provider protocols | OpenAI Responses and Chat Completions, OpenAI Images, Anthropic Messages, Google/Gemini Generate Content, xAI Responses, OpenRouter defaults, and custom OpenAI-compatible routes. | +| Provider protocols | OpenAI Responses, Chat Completions, Images Generations, and Images Edits; Anthropic Messages; Google/Gemini Generate Content and Image Generation; xAI Responses, Images Generations, and Images Edits; OpenRouter defaults; and custom OpenAI-compatible routes. | | Request governance | Protocol-aware request assembly, user option allowlists and denylists, system-protected fields, previous-response continuation where supported, and context snapshots for review. | | Files and RAG | File upload, preview, download, deletion, quota control, MIME detection, text extraction, OCR, full-context injection, image context, chunking, embeddings, and semantic retrieval. | | Memory and context | Message-window truncation, token-budget truncation, context compression, conversation memory, long-term user memory, RAG evidence records, and prompt trace inspection. | diff --git a/README.zh-CN.md b/README.zh-CN.md index 15a9d04f..1ed2cf26 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -34,9 +34,9 @@ DEEIX Chat 为团队提供统一的 AI 工作台，用一个清晰的使用入 | 模块 | 能力 | | --- | --- | | 对话体验 | 多分支会话、流式响应、重试、编辑、反馈、公开分享、克隆分享会话、富文本 Markdown、文件卡片、模型元信息、用量明细和执行链路。 | -| 媒体生成 | 独立的图片生成和图片编辑链路，按任务类型路由到 OpenAI Images 兼容协议，生成结果统一入库为文件，支持预览、下载和独立运行记录。 | +| 媒体生成 | 独立的图片生成和图片编辑链路，按任务类型路由到 OpenAI、Google 和 xAI 的原生图片协议，生成结果统一入库为文件，支持预览、下载和独立运行记录。 | | 模型控制面 | 平台模型目录、上游渠道、真实上游模型、路由绑定、优先级/权重路由、能力 JSON、展示顺序、厂商映射、自动图标和熔断状态。 | -| 协议适配 | OpenAI Responses 与 Chat Completions、OpenAI Images、Anthropic Messages、Google/Gemini Generate Content、xAI Responses、OpenRouter 默认协议和自定义 OpenAI 兼容路由。 | +| 协议适配 | OpenAI Responses、Chat Completions、Images Generations 和 Images Edits，Anthropic Messages，Google/Gemini Generate Content 和 Image Generation，xAI Responses、Images Generations 和 Images Edits，OpenRouter 默认协议和自定义 OpenAI 兼容路由。 | | 请求治理 | 按协议组装上游请求，支持用户参数白名单/黑名单、系统保护字段、协议支持时的 previous response 续接，以及可回看的上下文快照。 | | 文件与 RAG | 文件上传、预览、下载、删除、配额控制、MIME 探测、文本提取、OCR、全文上下文注入、图片上下文、分片、向量嵌入和语义检索。 | | 记忆与上下文 | 消息数截断、Token 预算截断、上下文压缩、会话记忆、用户长期记忆、RAG 证据记录和提示词链路查看。 | diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index 0d2386da..b9693fb2 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -9,7 +9,7 @@ import ( ) // @title DEEIX Chat API -// @version 0.1.0 +// @version 0.1.1 // @description DEEIX Chat 后端 API 文档 // @BasePath /api/v1 // @securityDefinitions.apikey BearerAuth diff --git a/backend/docs/docs.go b/backend/docs/docs.go index 51eef74a..ffb0deaa 100644 --- a/backend/docs/docs.go +++ b/backend/docs/docs.go @@ -9049,6 +9049,12 @@ const docTemplate = `{ "type": "string", "maxLength": 64 }, + "protocols": { + "type": "array", + "items": { + "type": "string" + } + }, "status": { "type": "string", "enum": [ @@ -9075,12 +9081,24 @@ const docTemplate = `{ "createdRoute": { "type": "boolean" }, + "createdRoutes": { + "type": "integer" + }, "error": { "type": "string" }, + "existingRoutes": { + "type": "integer" + }, "platformModelName": { "type": "string" }, + "protocols": { + "type": "array", + "items": { + "type": "string" + } + }, "status": { "type": "string" }, @@ -9908,6 +9926,12 @@ const docTemplate = `{ "suggestedProtocol": { "type": "string" }, + "suggestedProtocols": { + "type": "array", + "items": { + "type": "string" + } + }, "upstreamModelName": { "type": "string" }, @@ -11647,7 +11671,7 @@ const docTemplate = `{ // SwaggerInfo holds exported Swagger Info so clients can modify it var SwaggerInfo = &swag.Spec{ - Version: "0.1.0", + Version: "0.1.1", Host: "", BasePath: "/api/v1", Schemes: []string{}, diff --git a/backend/docs/swagger.json b/backend/docs/swagger.json index 05dc35da..4c6710e6 100644 --- a/backend/docs/swagger.json +++ b/backend/docs/swagger.json @@ -4,7 +4,7 @@ "description": "DEEIX Chat 后端 API 文档", "title": "DEEIX Chat API", "contact": {}, - "version": "0.1.0" + "version": "0.1.1" }, "basePath": "/api/v1", "paths": { @@ -9042,6 +9042,12 @@ "type": "string", "maxLength": 64 }, + "protocols": { + "type": "array", + "items": { + "type": "string" + } + }, "status": { "type": "string", "enum": [ @@ -9068,12 +9074,24 @@ "createdRoute": { "type": "boolean" }, + "createdRoutes": { + "type": "integer" + }, "error": { "type": "string" }, + "existingRoutes": { + "type": "integer" + }, "platformModelName": { "type": "string" }, + "protocols": { + "type": "array", + "items": { + "type": "string" + } + }, "status": { "type": "string" }, @@ -9901,6 +9919,12 @@ "suggestedProtocol": { "type": "string" }, + "suggestedProtocols": { + "type": "array", + "items": { + "type": "string" + } + }, "upstreamModelName": { "type": "string" }, diff --git a/backend/docs/swagger.yaml b/backend/docs/swagger.yaml index dd2a6006..d4edf25f 100644 --- a/backend/docs/swagger.yaml +++ b/backend/docs/swagger.yaml @@ -1649,6 +1649,10 @@ definitions: protocol: maxLength: 64 type: string + protocols: + items: + type: string + type: array status: enum: - active @@ -1670,10 +1674,18 @@ definitions: type: boolean createdRoute: type: boolean + createdRoutes: + type: integer error: type: string + existingRoutes: + type: integer platformModelName: type: string + protocols: + items: + type: string + type: array status: type: string upstreamModelName: @@ -2226,6 +2238,10 @@ definitions: type: string suggestedProtocol: type: string + suggestedProtocols: + items: + type: string + type: array upstreamModelName: type: string upstreamModelStatus: @@ -3371,7 +3387,7 @@ info: contact: {} description: DEEIX Chat 后端 API 文档 title: DEEIX Chat API - version: 0.1.0 + version: 0.1.1 paths: /admin/audit-logs: get: diff --git a/backend/internal/application/channel/dto.go b/backend/internal/application/channel/dto.go index 7d96a283..064788cf 100644 --- a/backend/internal/application/channel/dto.go +++ b/backend/internal/application/channel/dto.go @@ -40,6 +40,7 @@ type UpstreamRemoteModelView struct { SuggestedPlatformModelName string SuggestedKindsJSON string SuggestedProtocol string + SuggestedProtocols []string BindingCode string BoundPlatformModels []string UpstreamModelStatus string @@ -85,6 +86,9 @@ type ImportUpstreamModelResultView struct { BindingCode string Status string CreatedRoute bool + CreatedRoutes int + ExistingRoutes int + Protocols []string CreatedPlatform bool Error string } diff --git a/backend/internal/application/channel/errs.go b/backend/internal/application/channel/errs.go index 46293002..2211cfb8 100644 --- a/backend/internal/application/channel/errs.go +++ b/backend/internal/application/channel/errs.go @@ -41,6 +41,8 @@ var ( ErrInvalidModelOrder = errors.New("invalid model order") // ErrProtocolRequired 无法通过瀑布规则推断协议。 ErrProtocolRequired = errors.New("protocol required") + // ErrInvalidRouteProtocolCombination 路由协议组合无效。 + ErrInvalidRouteProtocolCombination = errors.New("invalid route protocol combination") // ErrUpstreamModelNotFound 上游模型路由绑定不存在。 ErrUpstreamModelNotFound = repository.ErrUpstreamModelNotFound // ErrUpstreamModelConflict 上游模型路由绑定冲突。 diff --git a/backend/internal/application/channel/input.go b/backend/internal/application/channel/input.go index a34df5ed..880bde54 100644 --- a/backend/internal/application/channel/input.go +++ b/backend/internal/application/channel/input.go @@ -97,6 +97,7 @@ type ImportUpstreamModelItemInput struct { PlatformModelName string UpstreamModelName string Protocol string + Protocols []string KindsJSON string Status string Priority int diff --git a/backend/internal/application/channel/model_catalog.go b/backend/internal/application/channel/model_catalog.go index 6d613de5..69b8dc3a 100644 --- a/backend/internal/application/channel/model_catalog.go +++ b/backend/internal/application/channel/model_catalog.go @@ -33,6 +33,7 @@ const ( protocolOpenAIVideoGenerations = "openai_video_generations" protocolGoogleImageGeneration = llm.AdapterGoogleImageGeneration protocolXAIImage = llm.AdapterXAIImage + protocolXAIImageEdits = llm.AdapterXAIImageEdits ) var protocolDefaultKindOrder = []string{ @@ -121,15 +122,17 @@ func systemFallbackProtocols(compatible string) map[string]string { } case compatibleGoogle: return map[string]string{ - modelKindChat: llm.AdapterGoogleGenerateContent, - modelKindAudio: llm.AdapterGoogleGenerateContent, - modelKindImageGen: protocolGoogleImageGeneration, + modelKindChat: llm.AdapterGoogleGenerateContent, + modelKindAudio: llm.AdapterGoogleGenerateContent, + modelKindImageGen: protocolGoogleImageGeneration, + modelKindImageEdit: protocolGoogleImageGeneration, } case compatibleXAI: return map[string]string{ - modelKindChat: llm.AdapterXAIResponses, - modelKindAudio: llm.AdapterXAIResponses, - modelKindImageGen: protocolXAIImage, + modelKindChat: llm.AdapterXAIResponses, + modelKindAudio: llm.AdapterXAIResponses, + modelKindImageGen: protocolXAIImage, + modelKindImageEdit: protocolXAIImageEdits, } case compatibleOpenRouter: return map[string]string{ @@ -162,7 +165,8 @@ func isKnownProtocol(raw string) bool { protocolOpenAIImageEdits, protocolOpenAIVideoGenerations, protocolGoogleImageGeneration, - protocolXAIImage: + protocolXAIImage, + protocolXAIImageEdits: return true default: return false @@ -175,7 +179,7 @@ func resolveRouteProtocol(explicit string, upCompatible string, defaultsJSON str if !isKnownProtocol(protocol) { return "", ErrInvalidAdapter } - if kind != "" && !isProtocolAllowedForKind(kind, protocol) { + if !isProtocolAllowedForKinds(kindsJSON, protocol) { return "", ErrInvalidAdapter } return protocol, nil @@ -193,6 +197,116 @@ func resolveRouteProtocol(explicit string, upCompatible string, defaultsJSON str return "", ErrProtocolRequired } +// resolveRouteProtocols 解析批量导入时的协议列表，图片生成/编辑模型会按协议能力生成一到两条绑定。 +func resolveRouteProtocols(explicit []string, upCompatible string, defaultsJSON string, kindsJSON string) ([]string, error) { + protocols := make([]string, 0, len(explicit)) + seen := make(map[string]struct{}, len(explicit)) + for _, raw := range explicit { + value := strings.TrimSpace(strings.ToLower(raw)) + if value == "" { + continue + } + protocol, err := resolveRouteProtocol(value, upCompatible, defaultsJSON, kindsJSON) + if err != nil { + return nil, err + } + if _, ok := seen[protocol]; ok { + continue + } + seen[protocol] = struct{}{} + protocols = append(protocols, protocol) + } + if len(protocols) > 0 { + if !isSupportedRouteProtocolCombination(protocols) { + return nil, ErrInvalidRouteProtocolCombination + } + return protocols, nil + } + + kinds := parseKinds(kindsJSON) + if hasModelKind(kinds, modelKindImageGen) && hasModelKind(kinds, modelKindImageEdit) { + generationProtocol := defaultRouteProtocolForKind(upCompatible, defaultsJSON, modelKindImageGen) + editProtocol := defaultRouteProtocolForKind(upCompatible, defaultsJSON, modelKindImageEdit) + if generationProtocol != "" && editProtocol != "" { + protocols = uniqueRouteProtocols(generationProtocol, editProtocol) + if isSupportedRouteProtocolCombination(protocols) { + return protocols, nil + } + } + } + + protocol, err := resolveRouteProtocol("", upCompatible, defaultsJSON, kindsJSON) + if err != nil { + return nil, err + } + return []string{protocol}, nil +} + +// uniqueRouteProtocols 保留协议声明顺序，同时避免 Google 图片这类同协议双能力模型创建重复绑定。 +func uniqueRouteProtocols(values ...string) []string { + protocols := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, raw := range values { + protocol := strings.TrimSpace(strings.ToLower(raw)) + if protocol == "" { + continue + } + if _, ok := seen[protocol]; ok { + continue + } + seen[protocol] = struct{}{} + protocols = append(protocols, protocol) + } + return protocols +} + +// defaultRouteProtocolForKind 优先使用上游配置的类型默认协议，其次回退到兼容类型内置协议。 +func defaultRouteProtocolForKind(upCompatible string, defaultsJSON string, kind string) string { + if protocol := protocolDefaultForKind(defaultsJSON, kind); protocol != "" { + return protocol + } + return systemFallbackProtocols(upCompatible)[kind] +} + +func isProtocolAllowedForKinds(kindsJSON string, protocol string) bool { + kinds := parseKinds(kindsJSON) + if len(kinds) == 0 { + return true + } + for _, kind := range kinds { + if isProtocolAllowedForKind(kind, protocol) { + return true + } + } + return false +} + +// isSupportedRouteProtocolCombination 限制同一绑定 pair 只能单协议，或图片生成/编辑双协议。 +func isSupportedRouteProtocolCombination(protocols []string) bool { + seen := make(map[string]struct{}, len(protocols)) + for _, raw := range protocols { + protocol := strings.TrimSpace(strings.ToLower(raw)) + if protocol == "" { + continue + } + seen[protocol] = struct{}{} + } + if len(seen) <= 1 { + return true + } + if len(seen) != 2 { + return false + } + _, hasGeneration := seen[protocolOpenAIImageGenerations] + _, hasEdit := seen[protocolOpenAIImageEdits] + if hasGeneration && hasEdit { + return true + } + _, hasGeneration = seen[protocolXAIImage] + _, hasEdit = seen[protocolXAIImageEdits] + return hasGeneration && hasEdit +} + func protocolDefaultForKind(defaultsJSON string, kind string) string { defaults := make(map[string]*string) if err := json.Unmarshal([]byte(strings.TrimSpace(defaultsJSON)), &defaults); err != nil { @@ -236,7 +350,9 @@ func isProtocolAllowedForKind(kind string, protocol string) bool { } case modelKindImageEdit: switch protocol { - case protocolOpenAIImageEdits: + case protocolOpenAIImageEdits, + protocolGoogleImageGeneration, + protocolXAIImageEdits: return true default: return false @@ -276,7 +392,7 @@ func IsRouteAllowedForTask(taskType string, kindsJSON string, protocol string) b case TaskTypeImageGeneration: return isProtocolAllowedForKind(modelKindImageGen, protocol) case TaskTypeImageEdit: - return protocol == protocolOpenAIImageEdits + return isProtocolAllowedForKind(modelKindImageEdit, protocol) default: return isProtocolAllowedForKind(modelKindChat, protocol) || isProtocolAllowedForKind(modelKindAudio, protocol) } @@ -285,7 +401,7 @@ func IsRouteAllowedForTask(taskType string, kindsJSON string, protocol string) b case TaskTypeImageGeneration: return hasModelKind(kinds, modelKindImageGen) && isProtocolAllowedForKind(modelKindImageGen, protocol) case TaskTypeImageEdit: - return hasModelKind(kinds, modelKindImageEdit) && protocol == protocolOpenAIImageEdits + return hasModelKind(kinds, modelKindImageEdit) && isProtocolAllowedForKind(modelKindImageEdit, protocol) default: for _, kind := range kinds { if (kind == modelKindChat || kind == modelKindAudio) && isProtocolAllowedForKind(kind, protocol) { @@ -321,9 +437,10 @@ func primaryKindFromKinds(kindsJSON string) string { func inferKindsJSON(platformModelName string) string { code := strings.ToLower(strings.TrimSpace(platformModelName)) switch { - case strings.HasPrefix(code, "gpt-image-"), code == "chatgpt-image-latest", code == "dall-e-2": + case strings.HasPrefix(code, "gpt-image-"), code == "chatgpt-image-latest", code == "dall-e-2", + isGeminiImageGenerationModel(code), isXAIImageGenerationModel(code): return `["image_gen","image_edit"]` - case code == "dall-e-3", strings.HasPrefix(code, "imagen-"), isGeminiImageGenerationModel(code), isXAIImageGenerationModel(code): + case code == "dall-e-3", strings.HasPrefix(code, "imagen-"): return `["image_gen"]` case code == "sora", code == "veo-2", strings.HasPrefix(code, "kling"): return `["video_gen"]` diff --git a/backend/internal/application/channel/model_catalog_test.go b/backend/internal/application/channel/model_catalog_test.go index 5121d29c..8607c68e 100644 --- a/backend/internal/application/channel/model_catalog_test.go +++ b/backend/internal/application/channel/model_catalog_test.go @@ -47,7 +47,10 @@ func TestProtocolDefaultsForXAIUsesXAIResponsesForConversationKinds(t *testing.T if defaults[modelKindImageGen] != "xai_image" { t.Fatalf("expected xAI image default, got %q in %s", defaults[modelKindImageGen], raw) } - for _, kind := range []string{modelKindImageEdit, modelKindVideoGen} { + if defaults[modelKindImageEdit] != "xai_image_edits" { + t.Fatalf("expected xAI image edit default, got %q in %s", defaults[modelKindImageEdit], raw) + } + for _, kind := range []string{modelKindVideoGen} { if _, ok := defaults[kind]; ok { t.Fatalf("unexpected xAI default protocol for %s in %s", kind, raw) } @@ -94,6 +97,9 @@ func TestProtocolDefaultsForGoogleUsesGoogleImageGeneration(t *testing.T) { if defaults[modelKindImageGen] != "google_image_generation" { t.Fatalf("expected Google image generation default, got %q in %s", defaults[modelKindImageGen], raw) } + if defaults[modelKindImageEdit] != "google_image_generation" { + t.Fatalf("expected Google image edit default, got %q in %s", defaults[modelKindImageEdit], raw) + } } func TestNormalizeCompatibleOnlyAllowsSupportedUpstreamProviders(t *testing.T) { @@ -289,8 +295,8 @@ func TestInferKindsJSONRecognizesGeminiImageModels(t *testing.T) { "gemini-3.1-flash-image-preview", "gemini-3-pro-image-preview", } { - if got := inferKindsJSON(modelName); got != `["image_gen"]` { - t.Fatalf("expected %s to infer image generation kind, got %s", modelName, got) + if got := inferKindsJSON(modelName); got != `["image_gen","image_edit"]` { + t.Fatalf("expected %s to infer image generation and edit kinds, got %s", modelName, got) } } } @@ -302,8 +308,8 @@ func TestInferKindsJSONRecognizesXAIImageModels(t *testing.T) { "grok-imagine-image-pro", "grok-imagine-image-preview", } { - if got := inferKindsJSON(modelName); got != `["image_gen"]` { - t.Fatalf("expected %s to infer image generation kind, got %s", modelName, got) + if got := inferKindsJSON(modelName); got != `["image_gen","image_edit"]` { + t.Fatalf("expected %s to infer image generation and edit kinds, got %s", modelName, got) } } } @@ -343,6 +349,116 @@ func TestResolveRouteProtocolRejectsExplicitProtocolKindMismatch(t *testing.T) { } } +func TestResolveRouteProtocolAcceptsExplicitProtocolForAnyDeclaredKind(t *testing.T) { + resolved, err := resolveRouteProtocol("openai_image_edits", compatibleOpenAI, "", `["image_gen","image_edit"]`) + if err != nil { + t.Fatalf("expected explicit image edit protocol to be accepted for dual-kind image model: %v", err) + } + if resolved != "openai_image_edits" { + t.Fatalf("expected openai_image_edits, got %q", resolved) + } +} + +func TestSupportedRouteProtocolCombinationOnlyAllowsSameProviderImagePair(t *testing.T) { + tests := []struct { + name string + protocols []string + want bool + }{ + {name: "single chat", protocols: []string{"openai_responses"}, want: true}, + {name: "single image generation", protocols: []string{"openai_image_generations"}, want: true}, + {name: "openai image generation and edit", protocols: []string{"openai_image_generations", "openai_image_edits"}, want: true}, + {name: "xai image generation and edit", protocols: []string{"xai_image", "xai_image_edits"}, want: true}, + {name: "duplicate protocol", protocols: []string{"openai_responses", "openai_responses"}, want: true}, + {name: "two chat protocols", protocols: []string{"openai_responses", "openai_chat_completions"}, want: false}, + {name: "image generation with chat", protocols: []string{"openai_image_generations", "openai_responses"}, want: false}, + {name: "mixed provider image pair", protocols: []string{"openai_image_generations", "xai_image_edits"}, want: false}, + {name: "three protocols", protocols: []string{"openai_image_generations", "openai_image_edits", "openai_responses"}, want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isSupportedRouteProtocolCombination(tt.protocols); got != tt.want { + t.Fatalf("expected %v, got %v", tt.want, got) + } + }) + } +} + +func TestResolveRouteProtocolsExpandsOpenAIDualImageKinds(t *testing.T) { + protocols, err := resolveRouteProtocols(nil, compatibleOpenAI, "", `["image_gen","image_edit"]`) + if err != nil { + t.Fatalf("resolve route protocols: %v", err) + } + expected := []string{"openai_image_generations", "openai_image_edits"} + if len(protocols) != len(expected) { + t.Fatalf("expected %d protocols, got %#v", len(expected), protocols) + } + for i, expectedProtocol := range expected { + if protocols[i] != expectedProtocol { + t.Fatalf("expected protocol %d to be %q, got %#v", i, expectedProtocol, protocols) + } + } +} + +func TestResolveRouteProtocolsKeepsSingleGoogleProtocolForDualImageKinds(t *testing.T) { + protocols, err := resolveRouteProtocols(nil, compatibleGoogle, "", `["image_gen","image_edit"]`) + if err != nil { + t.Fatalf("resolve route protocols: %v", err) + } + if len(protocols) != 1 || protocols[0] != "google_image_generation" { + t.Fatalf("expected single Google image protocol, got %#v", protocols) + } +} + +func TestResolveRouteProtocolsExpandsXAIDualImageKinds(t *testing.T) { + protocols, err := resolveRouteProtocols(nil, compatibleXAI, "", `["image_gen","image_edit"]`) + if err != nil { + t.Fatalf("resolve route protocols: %v", err) + } + expected := []string{"xai_image", "xai_image_edits"} + if len(protocols) != len(expected) { + t.Fatalf("expected %d protocols, got %#v", len(expected), protocols) + } + for i, expectedProtocol := range expected { + if protocols[i] != expectedProtocol { + t.Fatalf("expected protocol %d to be %q, got %#v", i, expectedProtocol, protocols) + } + } +} + +func TestResolveRouteProtocolsKeepsSingleProtocolForGenerationOnlyModels(t *testing.T) { + tests := []struct { + name string + compatible string + kindsJSON string + expected string + }{ + {name: "openai generation", compatible: compatibleOpenAI, kindsJSON: `["image_gen"]`, expected: "openai_image_generations"}, + {name: "google generation", compatible: compatibleGoogle, kindsJSON: `["image_gen"]`, expected: "google_image_generation"}, + {name: "xai generation", compatible: compatibleXAI, kindsJSON: `["image_gen"]`, expected: "xai_image"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + protocols, err := resolveRouteProtocols(nil, tt.compatible, "", tt.kindsJSON) + if err != nil { + t.Fatalf("resolve route protocols: %v", err) + } + if len(protocols) != 1 || protocols[0] != tt.expected { + t.Fatalf("expected [%s], got %#v", tt.expected, protocols) + } + }) + } +} + +func TestResolveRouteProtocolsRejectsUnsupportedExplicitMultiProtocol(t *testing.T) { + _, err := resolveRouteProtocols([]string{"openai_responses", "openai_chat_completions"}, compatibleOpenAI, "", `["chat"]`) + if !errors.Is(err, ErrInvalidRouteProtocolCombination) { + t.Fatalf("expected ErrInvalidRouteProtocolCombination, got %v", err) + } +} + func TestResolveRouteProtocolUsesExplicitConversationProtocolAsSourceOfTruth(t *testing.T) { for _, tc := range []struct { compatible string @@ -383,6 +499,15 @@ func TestIsRouteAllowedForTaskSeparatesChatAndImageProtocols(t *testing.T) { if !IsRouteAllowedForTask(TaskTypeImageEdit, `["image_edit"]`, "openai_image_edits") { t.Fatalf("expected image edit task to allow image edit protocol") } + if !IsRouteAllowedForTask(TaskTypeImageEdit, `["image_edit"]`, "google_image_generation") { + t.Fatalf("expected image edit task to allow Google image protocol") + } + if !IsRouteAllowedForTask(TaskTypeImageEdit, `["image_edit"]`, "xai_image_edits") { + t.Fatalf("expected image edit task to allow xAI image edits protocol") + } + if IsRouteAllowedForTask(TaskTypeImageEdit, `["image_edit"]`, "xai_image") { + t.Fatalf("expected image edit task to reject xAI image generation protocol") + } } func TestDefaultRouteModelMatchesTaskFiltersByKind(t *testing.T) { diff --git a/backend/internal/application/channel/service_model.go b/backend/internal/application/channel/service_model.go index 7e6d656c..20b82d15 100644 --- a/backend/internal/application/channel/service_model.go +++ b/backend/internal/application/channel/service_model.go @@ -436,10 +436,20 @@ func (s *Service) UpdateModelUpstreamSource(ctx context.Context, modelID uint, r if err != nil { return nil, err } + source, err := s.repo.GetModelUpstreamSourceByRouteID(ctx, modelItem.PlatformModelName, routeID) + if err != nil { + return nil, err + } updateInput := repository.UpdateChannelPlatformRouteInput{} if input.Protocol != nil { - protocol := strings.TrimSpace(*input.Protocol) + protocol, err := normalizeProtocol(*input.Protocol) + if err != nil { + return nil, err + } + if err := s.validateRouteProtocolCombination(ctx, source.UpstreamID, modelItem.ID, source.UpstreamModelID, routeID, protocol); err != nil { + return nil, err + } updateInput.Protocol = &protocol } if input.Status != nil { @@ -455,47 +465,25 @@ func (s *Service) UpdateModelUpstreamSource(ctx context.Context, modelID uint, r updateInput.Weight = &weight } - allSources, _, listErr := s.repo.ListModelUpstreamSources(ctx, modelItem.PlatformModelName, 0, 2000) - if listErr != nil { - return nil, listErr - } - var targetUpstreamID uint - for _, src := range allSources { - if src.ID == routeID { - targetUpstreamID = src.UpstreamID - break - } - } - if targetUpstreamID == 0 { - return nil, ErrUpstreamModelNotFound - } - if updateInput.IsZero() { - for _, src := range allSources { - if src.ID == routeID { - view := toModelUpstreamSourceView(src) - return &view, nil - } - } - return nil, ErrUpstreamModelNotFound + view := toModelUpstreamSourceView(*source) + return &view, nil } - if err := s.repo.UpdatePlatformModelRouteByID(ctx, routeID, targetUpstreamID, updateInput); err != nil { + if err := s.repo.UpdatePlatformModelRouteByID(ctx, routeID, source.UpstreamID, updateInput); err != nil { + if isDuplicateKeyError(err) { + return nil, ErrUpstreamModelConflict + } return nil, err } s.InvalidateModelCatalog() - allSources, _, err = s.repo.ListModelUpstreamSources(ctx, modelItem.PlatformModelName, 0, 500) + source, err = s.repo.GetModelUpstreamSourceByRouteID(ctx, modelItem.PlatformModelName, routeID) if err != nil { return nil, err } - for _, src := range allSources { - if src.ID == routeID { - view := toModelUpstreamSourceView(src) - return &view, nil - } - } - return nil, ErrUpstreamModelNotFound + view := toModelUpstreamSourceView(*source) + return &view, nil } // --------------------------------------------------------------------------- diff --git a/backend/internal/application/channel/service_sync.go b/backend/internal/application/channel/service_sync.go index 93d9a281..3b1c5977 100644 --- a/backend/internal/application/channel/service_sync.go +++ b/backend/internal/application/channel/service_sync.go @@ -11,7 +11,6 @@ import ( domainchannel "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/channel" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/llm" - "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" "go.uber.org/zap" ) @@ -34,7 +33,16 @@ func (s *Service) ListRemoteModels(ctx context.Context, upstreamID uint) (*Upstr return nil, err } - rows, _, _ := s.repo.ListUpstreamModels(ctx, upstreamID, repository.ListChannelUpstreamModelsInput{Offset: 0, Limit: 5000}) + remoteNames := make([]string, 0, len(items)) + for _, item := range items { + if name := strings.TrimSpace(item.ID); name != "" { + remoteNames = append(remoteNames, name) + } + } + rows, err := s.repo.ListUpstreamModelsByNames(ctx, upstreamID, remoteNames) + if err != nil { + return nil, err + } existingByName := make(map[string]repositoryUpstreamModelSnapshot, len(rows)) for _, row := range rows { name := strings.TrimSpace(row.UpstreamModelName) @@ -57,13 +65,18 @@ func (s *Service) ListRemoteModels(ctx context.Context, upstreamID uint) (*Upstr continue } kindsJSON := inferKindsJSON(name) - suggestedProtocol, _ := resolveRouteProtocol("", upstreamItem.Compatible, upstreamItem.ProtocolDefaultsJSON, kindsJSON) + suggestedProtocols, _ := resolveRouteProtocols(nil, upstreamItem.Compatible, upstreamItem.ProtocolDefaultsJSON, kindsJSON) + suggestedProtocol := "" + if len(suggestedProtocols) > 0 { + suggestedProtocol = suggestedProtocols[0] + } snapshot, alreadySynced := existingByName[name] views = append(views, UpstreamRemoteModelView{ UpstreamModelName: name, SuggestedPlatformModelName: name, SuggestedKindsJSON: kindsJSON, SuggestedProtocol: suggestedProtocol, + SuggestedProtocols: suggestedProtocols, BindingCode: snapshot.BindingCode, BoundPlatformModels: snapshot.BoundPlatformModels, UpstreamModelStatus: snapshot.Status, @@ -157,7 +170,8 @@ func (s *Service) SyncUpstreamModels(ctx context.Context, upstreamID uint) (*Syn // ImportUpstreamModels 批量把上游真实模型绑定到平台模型。 func (s *Service) ImportUpstreamModels(ctx context.Context, upstreamID uint, input ImportUpstreamModelsInput) (*ImportUpstreamModelsData, error) { - if _, err := s.repo.GetUpstreamByID(ctx, upstreamID); err != nil { + upstreamItem, err := s.repo.GetUpstreamByID(ctx, upstreamID) + if err != nil { if errors.Is(err, ErrUpstreamNotFound) { return nil, ErrUpstreamNotFound } @@ -169,7 +183,7 @@ func (s *Service) ImportUpstreamModels(ctx context.Context, upstreamID uint, inp Results: make([]ImportUpstreamModelResultView, 0, len(input.Items)), } for _, item := range input.Items { - imported, importErr := s.importSingleUpstreamModel(ctx, upstreamID, item) + imported, importErr := s.importSingleUpstreamModel(ctx, upstreamItem, item) if importErr != nil { result.FailedCount++ result.Results = append(result.Results, ImportUpstreamModelResultView{ @@ -182,11 +196,10 @@ func (s *Service) ImportUpstreamModels(ctx context.Context, upstreamID uint, inp } result.ImportedCount++ status := ImportUpstreamModelStatusExisting - if imported.CreatedRoute { - result.CreatedRoutes++ + result.CreatedRoutes += imported.CreatedRoutes + result.ExistingRoutes += imported.ExistingRoutes + if imported.CreatedRoutes > 0 { status = ImportUpstreamModelStatusCreated - } else { - result.ExistingRoutes++ } if imported.CreatedPlatform { result.CreatedPlatform++ @@ -290,7 +303,7 @@ func (s *Service) syncSingleUpstreamModel(ctx context.Context, up *domainchannel }, nil } -func (s *Service) importSingleUpstreamModel(ctx context.Context, upstreamID uint, input ImportUpstreamModelItemInput) (ImportUpstreamModelResultView, error) { +func (s *Service) importSingleUpstreamModel(ctx context.Context, upstreamItem *domainchannel.Upstream, input ImportUpstreamModelItemInput) (ImportUpstreamModelResultView, error) { platformModelName, err := normalizePlatformModelName(input.PlatformModelName) if err != nil { return ImportUpstreamModelResultView{}, err @@ -304,41 +317,54 @@ func (s *Service) importSingleUpstreamModel(ctx context.Context, upstreamID uint if platformErr != nil && !createdPlatform { return ImportUpstreamModelResultView{}, platformErr } - createdRoute := !s.routeExists(ctx, upstreamID, platformModelName, upstreamModelName) - view, err := s.UpsertUpstreamModel(ctx, upstreamID, UpsertUpstreamModelInput{ - PlatformModelName: platformModelName, - UpstreamModelName: upstreamModelName, - Protocol: input.Protocol, - KindsJSON: input.KindsJSON, - Status: input.Status, - Priority: input.Priority, - Weight: 1, - Source: "import", - }) + kindsJSON := strings.TrimSpace(input.KindsJSON) + if kindsJSON == "" { + kindsJSON = inferKindsJSON(platformModelName) + } + explicitProtocols := append([]string{}, input.Protocols...) + if len(explicitProtocols) == 0 && strings.TrimSpace(input.Protocol) != "" { + explicitProtocols = append(explicitProtocols, input.Protocol) + } + protocols, err := resolveRouteProtocols(explicitProtocols, upstreamItem.Compatible, upstreamItem.ProtocolDefaultsJSON, kindsJSON) if err != nil { return ImportUpstreamModelResultView{}, err } - return ImportUpstreamModelResultView{ - UpstreamModelName: view.UpstreamModelName, - PlatformModelName: view.PlatformModelName, - BindingCode: view.BindingCode, - CreatedRoute: createdRoute, + result := ImportUpstreamModelResultView{ + UpstreamModelName: upstreamModelName, + PlatformModelName: platformModelName, CreatedPlatform: createdPlatform, - }, nil -} - -func (s *Service) routeExists(ctx context.Context, upstreamID uint, platformModelName string, upstreamModelName string) bool { - rows, _, err := s.repo.ListUpstreamModels(ctx, upstreamID, repository.ListChannelUpstreamModelsInput{Offset: 0, Limit: 5000}) - if err != nil { - return false + Protocols: protocols, } - for _, row := range rows { - if strings.TrimSpace(row.PlatformModelName) == platformModelName && - strings.TrimSpace(row.UpstreamModelName) == upstreamModelName && - row.RouteID > 0 { - return true + for _, protocol := range protocols { + createdRoute := !s.routeExists(ctx, upstreamItem.ID, platformModelName, upstreamModelName, protocol) + view, err := s.UpsertUpstreamModel(ctx, upstreamItem.ID, UpsertUpstreamModelInput{ + PlatformModelName: platformModelName, + UpstreamModelName: upstreamModelName, + Protocol: protocol, + KindsJSON: kindsJSON, + Status: input.Status, + Priority: input.Priority, + Weight: 1, + Source: "import", + }) + if err != nil { + return ImportUpstreamModelResultView{}, err + } + if result.BindingCode == "" { + result.BindingCode = view.BindingCode + } + if createdRoute { + result.CreatedRoutes++ + result.CreatedRoute = true + } else { + result.ExistingRoutes++ } } - return false + return result, nil +} + +func (s *Service) routeExists(ctx context.Context, upstreamID uint, platformModelName string, upstreamModelName string, protocol string) bool { + _, err := s.repo.GetUpstreamModelRouteByNames(ctx, upstreamID, platformModelName, upstreamModelName, protocol) + return err == nil } diff --git a/backend/internal/application/channel/service_upstream_model.go b/backend/internal/application/channel/service_upstream_model.go index 33ceb2d1..cdfc7590 100644 --- a/backend/internal/application/channel/service_upstream_model.go +++ b/backend/internal/application/channel/service_upstream_model.go @@ -68,7 +68,9 @@ func (s *Service) UpsertUpstreamModel(ctx context.Context, upstreamID uint, inpu return nil, ErrInvalidJSONConfig } - kindsJSON := strings.TrimSpace(input.KindsJSON) + rawKindsJSON := strings.TrimSpace(input.KindsJSON) + kindsExplicit := rawKindsJSON != "" + kindsJSON := rawKindsJSON if kindsJSON == "" { kindsJSON = inferKindsJSON(platformModelName) } @@ -81,16 +83,25 @@ func (s *Service) UpsertUpstreamModel(ctx context.Context, upstreamID uint, inpu return nil, err } - platformModel, _, err := s.ensurePlatformModel(ctx, platformModelName, kindsJSON, upstreamModelName) + platformModel, platformModelCreated, err := s.ensurePlatformModel(ctx, platformModelName, kindsJSON, upstreamModelName) if err != nil { return nil, err } + if !platformModelCreated && kindsExplicit && strings.TrimSpace(platformModel.KindsJSON) != kindsJSON { + if err := s.repo.UpdateModel(ctx, platformModel.ID, repository.UpdateChannelModelInput{KindsJSON: &kindsJSON}); err != nil { + return nil, err + } + platformModel.KindsJSON = kindsJSON + } upstreamModelVendor := normalizeUpstreamModelVendor("", upstreamModelName, upstream.Name, upstream.BaseURL) upstreamModelIcon := normalizeModelIcon("", upstreamModelVendor, upstreamModelName) upstreamModel, err := s.upsertUpstreamCatalogModel(ctx, upstream.ID, upstreamModelName, protocol, kindsJSON, upstreamModelVendor, upstreamModelIcon, "active", normalizeSource(input.Source), "{}") if err != nil { return nil, err } + if err := s.validateRouteProtocolCombination(ctx, upstream.ID, platformModel.ID, upstreamModel.ID, input.RouteID, protocol); err != nil { + return nil, err + } route := &domainchannel.PlatformModelRoute{ PlatformModelID: platformModel.ID, @@ -140,6 +151,33 @@ func (s *Service) UpsertUpstreamModel(ctx context.Context, upstreamID uint, inpu return s.findUpstreamModelViewByRoute(ctx, upstream.ID, route.ID, upstreamModel.ID) } +func (s *Service) validateRouteProtocolCombination( + ctx context.Context, + upstreamID uint, + platformModelID uint, + upstreamModelID uint, + routeID uint, + protocol string, +) error { + // 同一个平台模型到同一个上游真实模型只允许单协议，或同厂商图片生成/编辑成对组合。 + routes, err := s.repo.ListPlatformModelRoutesByPair(ctx, upstreamID, platformModelID, upstreamModelID) + if err != nil { + return err + } + protocols := make([]string, 0, len(routes)+1) + for _, route := range routes { + if route.ID == routeID { + continue + } + protocols = append(protocols, route.Protocol) + } + protocols = append(protocols, protocol) + if !isSupportedRouteProtocolCombination(protocols) { + return ErrInvalidRouteProtocolCombination + } + return nil +} + func (s *Service) ensurePlatformModel(ctx context.Context, platformModelName string, kindsJSON string, candidates ...string) (*domainchannel.PlatformModel, bool, error) { if item, err := s.repo.GetModelByName(ctx, platformModelName); err == nil { return item, false, nil @@ -226,21 +264,15 @@ func (s *Service) upsertUpstreamCatalogModel( } func (s *Service) findUpstreamModelViewByRoute(ctx context.Context, upstreamID uint, routeID uint, upstreamModelID uint) (*UpstreamModelView, error) { - rows, _, err := s.repo.ListUpstreamModels(ctx, upstreamID, repository.ListChannelUpstreamModelsInput{Offset: 0, Limit: 5000}) + row, err := s.repo.GetUpstreamModelRouteByID(ctx, upstreamID, routeID) if err != nil { return nil, err } - for _, row := range rows { - if routeID > 0 && row.RouteID == routeID { - view := toUpstreamModelView(row) - return &view, nil - } - if routeID == 0 && row.ID == upstreamModelID { - view := toUpstreamModelView(row) - return &view, nil - } + if upstreamModelID > 0 && row.ID != upstreamModelID { + return nil, ErrUpstreamModelNotFound } - return nil, ErrUpstreamModelNotFound + view := toUpstreamModelView(*row) + return &view, nil } // DeleteUpstreamModel 删除平台路由绑定，保留上游真实模型清单。 @@ -316,17 +348,12 @@ func (s *Service) ResetUpstreamModelCircuit(ctx context.Context, upstreamID uint } func (s *Service) routeBindingCode(ctx context.Context, upstreamID uint, routeID uint) (string, error) { - rows, _, err := s.repo.ListUpstreamModels(ctx, upstreamID, repository.ListChannelUpstreamModelsInput{Offset: 0, Limit: 5000}) + row, err := s.repo.GetUpstreamModelRouteByID(ctx, upstreamID, routeID) if err != nil { return "", err } - for _, row := range rows { - if row.RouteID == routeID { - if strings.TrimSpace(row.BindingCode) == "" { - return "", ErrUpstreamModelNotFound - } - return row.BindingCode, nil - } + if strings.TrimSpace(row.BindingCode) == "" { + return "", ErrUpstreamModelNotFound } - return "", ErrUpstreamModelNotFound + return row.BindingCode, nil } diff --git a/backend/internal/application/conversation/errs.go b/backend/internal/application/conversation/errs.go index 3987764d..003297eb 100644 --- a/backend/internal/application/conversation/errs.go +++ b/backend/internal/application/conversation/errs.go @@ -59,8 +59,18 @@ var ( ErrMessageGenerationCanceled = errors.New("message generation canceled") // ErrInvalidMediaGenerationTask 媒体生成任务类型或输入不合法。 ErrInvalidMediaGenerationTask = errors.New("invalid media generation task") + // ErrMediaImagePromptRequired 图片任务提示词不能为空。 + ErrMediaImagePromptRequired = errors.New("image prompt is required") + // ErrMediaImageGenerationRejectsInputs 图片生成任务不能携带输入图。 + ErrMediaImageGenerationRejectsInputs = errors.New("image generation does not accept input images") + // ErrMediaImageEditInputRequired 图片编辑任务必须携带至少一张输入图。 + ErrMediaImageEditInputRequired = errors.New("image edit requires at least one input image") + // ErrMediaImageEditTooManyInputs 图片编辑输入图数量超限。 + ErrMediaImageEditTooManyInputs = errors.New("too many image edit input images") + // ErrMediaImageEditInputInvalid 图片编辑输入图不合法。 + ErrMediaImageEditInputInvalid = errors.New("image edit input image is invalid") + // ErrMediaRouteProtocolMismatch 图片任务命中的路由协议与任务类型不匹配。 + ErrMediaRouteProtocolMismatch = errors.New("media route protocol does not match task") // ErrDuplicateMessageGenerationRun 表示客户端重复提交同一个生成 run。 ErrDuplicateMessageGenerationRun = errors.New("duplicate message generation run") - // ErrMediaImageEditNotImplemented 图片编辑协议尚未实现。 - ErrMediaImageEditNotImplemented = errors.New("image edit protocol not implemented") ) diff --git a/backend/internal/application/conversation/model_option_policy.go b/backend/internal/application/conversation/model_option_policy.go index 09e99e9e..5a91b6c4 100644 --- a/backend/internal/application/conversation/model_option_policy.go +++ b/backend/internal/application/conversation/model_option_policy.go @@ -290,7 +290,7 @@ func sanitizeModelOptionValues(options map[string]interface{}, protocolKey strin default: delete(options, "service_tier") } - case "openai_image_generations": + case "openai_image_generations", "openai_image_edits": value, ok := modelParamIntFromOption(options["partial_images"]) if !ok { delete(options, "partial_images") @@ -325,10 +325,14 @@ func modelOptionPolicyProtocolKey(protocol string) string { return "openai_chat_completions" case llm.AdapterOpenAIImageGenerations: return "openai_image_generations" + case llm.AdapterOpenAIImageEdits: + return "openai_image_edits" case llm.AdapterAnthropicMessages: return "anthropic_messages" case llm.AdapterXAIImage: return "xai_image" + case llm.AdapterXAIImageEdits: + return "xai_image_edits" case llm.AdapterXAIResponses: return "xai_responses" default: diff --git a/backend/internal/application/conversation/model_option_policy_test.go b/backend/internal/application/conversation/model_option_policy_test.go index 7588089f..5707d54a 100644 --- a/backend/internal/application/conversation/model_option_policy_test.go +++ b/backend/internal/application/conversation/model_option_policy_test.go @@ -279,6 +279,37 @@ func TestFilterModelOptionsOpenAIImageGenerationsAllowsImageParams(t *testing.T) } } +func TestFilterModelOptionsOpenAIImageEditsAllowsEditParams(t *testing.T) { + filtered := filterModelOptions(map[string]interface{}{ + "background": "transparent", + "input_fidelity": "high", + "n": 1, + "output_compression": 80, + "output_format": "webp", + "partial_images": 2, + "quality": "high", + "size": "1024x1024", + "prompt": "override", + "stream": true, + }, llm.AdapterOpenAIImageEdits, modelOptionPolicyConfig{ + Mode: modelOptionPolicyAllowlist, + AllowedPathsJSON: config.DefaultModelOptionAllowedPathsJSON(), + DeniedPathsJSON: config.DefaultModelOptionDeniedPathsJSON(), + }) + + if filtered["background"] != "transparent" || filtered["input_fidelity"] != "high" { + t.Fatalf("expected image edit params to pass, got %#v", filtered) + } + if filtered["partial_images"] != 2 || filtered["output_format"] != "webp" { + t.Fatalf("expected image edit output params to pass, got %#v", filtered) + } + for _, key := range []string{"prompt", "stream"} { + if _, ok := filtered[key]; ok { + t.Fatalf("expected %s to be hard denied, got %#v", key, filtered) + } + } +} + func TestFilterModelOptionsXAIImageAllowsImageParams(t *testing.T) { filtered := filterModelOptions(map[string]interface{}{ "aspect_ratio": "16:9", diff --git a/backend/internal/application/conversation/service_helpers.go b/backend/internal/application/conversation/service_helpers.go index b76f8bef..e4b6cc71 100644 --- a/backend/internal/application/conversation/service_helpers.go +++ b/backend/internal/application/conversation/service_helpers.go @@ -173,6 +173,18 @@ func classifyRunErrorCode(err error) string { return "upstream_empty_response" case errors.Is(err, ErrMessageGenerationCanceled): return "generation_canceled" + case errors.Is(err, ErrMediaImagePromptRequired): + return "media_image_prompt_required" + case errors.Is(err, ErrMediaImageGenerationRejectsInputs): + return "media_image_generation_rejects_inputs" + case errors.Is(err, ErrMediaImageEditInputRequired): + return "media_image_edit_input_required" + case errors.Is(err, ErrMediaImageEditTooManyInputs): + return "media_image_edit_too_many_inputs" + case errors.Is(err, ErrMediaImageEditInputInvalid): + return "media_image_edit_input_invalid" + case errors.Is(err, ErrMediaRouteProtocolMismatch): + return "media_route_protocol_mismatch" case errors.Is(err, ErrUpstreamRequestFailed): return "upstream_request_failed" default: diff --git a/backend/internal/application/conversation/service_media_generation.go b/backend/internal/application/conversation/service_media_generation.go index 97e52b23..1f4eb083 100644 --- a/backend/internal/application/conversation/service_media_generation.go +++ b/backend/internal/application/conversation/service_media_generation.go @@ -15,6 +15,7 @@ import ( model "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/conversation" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/infra/llm" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/pkg/traceid" + "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/repository" "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/shared/security" "github.com/google/uuid" "go.uber.org/zap" @@ -26,10 +27,12 @@ type MediaImageTaskType string const ( // MediaImageTaskGeneration 表示纯文本提示词生成图片任务。 MediaImageTaskGeneration MediaImageTaskType = "image_generation" - // MediaImageTaskEdit 表示基于输入图片的编辑任务，当前接口已预留但 adapter 尚未落地。 + // MediaImageTaskEdit 表示基于输入图片的编辑任务。 MediaImageTaskEdit MediaImageTaskType = "image_edit" ) +const maxMediaImageEditInputImages = 16 + // MediaImageInput 定义媒体图片任务的应用层入参。 type MediaImageInput struct { UserID uint @@ -41,6 +44,7 @@ type MediaImageInput struct { Options map[string]interface{} ClientRunID string FileIDs []string + MaskFileID string ParentMessagePublicID string SourceMessagePublicID string BranchReason string @@ -50,17 +54,17 @@ type MediaImageInput struct { // StreamMediaImage 执行图片生成任务并把结果保存为文件对象。 // 图片能力不复用聊天生成链路，只通过图片任务类型和图片协议路由。 func (s *Service) StreamMediaImage(ctx context.Context, input MediaImageInput) (*SendMessageResult, error) { - if input.TaskType == MediaImageTaskEdit { - return nil, ErrMediaImageEditNotImplemented - } - if input.TaskType != MediaImageTaskGeneration { + if input.TaskType != MediaImageTaskGeneration && input.TaskType != MediaImageTaskEdit { return nil, ErrInvalidMediaGenerationTask } if strings.TrimSpace(input.Prompt) == "" { - return nil, ErrInvalidMediaGenerationTask + return nil, ErrMediaImagePromptRequired } - if len(input.FileIDs) > 0 { - return nil, ErrInvalidMediaGenerationTask + if input.TaskType == MediaImageTaskGeneration && len(input.FileIDs) > 0 { + return nil, ErrMediaImageGenerationRejectsInputs + } + if input.TaskType == MediaImageTaskEdit && len(input.FileIDs) == 0 { + return nil, ErrMediaImageEditInputRequired } if s.routeResolver == nil || s.llmClient == nil { return nil, ErrModelRouteNotConfigured @@ -96,9 +100,15 @@ func (s *Service) StreamMediaImage(ctx context.Context, input MediaImageInput) ( if platformModelName == "" { return nil, ErrModelRouteNotConfigured } + taskRouteType := channel.TaskTypeImageGeneration + endpoint := llm.EndpointImageGenerations + if input.TaskType == MediaImageTaskEdit { + taskRouteType = channel.TaskTypeImageEdit + endpoint = llm.EndpointImageEdits + } route, err := s.routeResolver.ResolveRoute(ctx, channel.ResolveRouteInput{ PlatformModelName: platformModelName, - TaskType: channel.TaskTypeImageGeneration, + TaskType: taskRouteType, UserID: input.UserID, ConversationID: input.ConversationID, RequestID: strings.TrimSpace(input.RequestID), @@ -106,8 +116,11 @@ func (s *Service) StreamMediaImage(ctx context.Context, input MediaImageInput) ( if err != nil { return nil, ErrModelRouteNotConfigured } - if !llm.IsImageGenerationAdapter(route.Protocol) { - return nil, ErrInvalidMediaGenerationTask + if input.TaskType == MediaImageTaskGeneration && !llm.IsImageGenerationAdapter(route.Protocol) { + return nil, ErrMediaRouteProtocolMismatch + } + if input.TaskType == MediaImageTaskEdit && !llm.IsImageEditAdapter(route.Protocol) { + return nil, ErrMediaRouteProtocolMismatch } // 图片任务会把会话当前模型更新为实际执行的图片模型；标题、标签等内部文本任务会单独回退到聊天模型。 if strings.TrimSpace(conversation.Model) != strings.TrimSpace(route.PlatformModelName) { @@ -124,13 +137,23 @@ func (s *Service) StreamMediaImage(ctx context.Context, input MediaImageInput) ( return nil, err } + resolvedAttachments, imageEditParts, err := s.resolveMediaImageEditInputs(ctx, input) + if err != nil { + return nil, err + } + maskPart, err := s.resolveMediaImageEditMask(ctx, input.UserID, input.MaskFileID) + if err != nil { + return nil, err + } + attachmentsJSON := marshalAttachmentSnapshots(resolvedAttachments) + run := &model.Run{ RunID: runID, RequestID: strings.TrimSpace(input.RequestID), UserID: input.UserID, ConversationID: input.ConversationID, TaskType: string(input.TaskType), - Endpoint: llm.EndpointImageGenerations, + Endpoint: endpoint, Provider: strings.TrimSpace(conversation.Provider), ProviderProtocol: route.Protocol, UpstreamID: route.UpstreamID, @@ -173,39 +196,56 @@ func (s *Service) StreamMediaImage(ctx context.Context, input MediaImageInput) ( ParentMessageID: branchState.ParentMessageID, RunID: runID, Role: "user", - ContentType: "text", + ContentType: mediaImageUserContentType(input.TaskType), Content: strings.TrimSpace(input.Prompt), BranchReason: normalizedBranchReason, SourceMessageID: branchState.SourceMessageID, TokenUsage: estimateTokens(input.Prompt), InputTokens: estimateTokens(input.Prompt), Status: "success", - Attachments: "[]", - } - if err = s.repo.CreateMessage(ctx, userMessage); err != nil { - retErr = err - return nil, err + Attachments: attachmentsJSON, + } + userAttachmentRows := make([]model.Attachment, 0, len(resolvedAttachments)) + if len(resolvedAttachments) > 0 { + now := time.Now() + for _, item := range resolvedAttachments { + userAttachmentRows = append(userAttachmentRows, model.Attachment{ + ConversationID: input.ConversationID, + UserID: input.UserID, + FileID: strings.TrimSpace(item.FileID), + Kind: normalizeAttachmentKind(item.Kind, item.MimeType), + FileName: strings.TrimSpace(item.FileName), + MimeType: strings.TrimSpace(item.MimeType), + FileSize: item.FileSize, + SHA256: strings.TrimSpace(item.SHA256), + StoragePath: strings.TrimSpace(item.StoragePath), + Status: "active", + MetaJSON: strings.TrimSpace(item.MetaJSON), + UploadedAt: now, + }) + } } - userMessage.ParentPublicID = branchState.ParentPublicID - userMessage.SourcePublicID = branchState.SourcePublicID assistantMessage := &model.Message{ - ConversationID: input.ConversationID, - UserID: input.UserID, - PublicID: normalizePublicID(uuid.NewString()), - ParentMessageID: &userMessage.ID, - RunID: runID, - Role: "assistant", - ContentType: "image", - Content: "", - BranchReason: normalizedBranchReason, - Status: "pending", - Attachments: "[]", - } - if err = s.repo.CreateMessage(ctx, assistantMessage); err != nil { + ConversationID: input.ConversationID, + UserID: input.UserID, + PublicID: normalizePublicID(uuid.NewString()), + RunID: runID, + Role: "assistant", + ContentType: "image", + Content: "", + BranchReason: normalizedBranchReason, + Status: "pending", + Attachments: "[]", + } + // 媒体任务同样产生一个完整消息回合，初始本地写入必须原子提交。 + if err = s.repo.CreateMessagePairWithUserAttachments(ctx, userMessage, assistantMessage, userAttachmentRows); err != nil { retErr = err return nil, err } + userMessage.ParentPublicID = branchState.ParentPublicID + userMessage.SourcePublicID = branchState.SourcePublicID + assistantMessage.ParentPublicID = userMessage.PublicID traceRecorder := newMessageTraceRecorder(s, ctx, assistantMessage, input.OnEvent) defer func() { if retErr != nil && traceRecorder != nil { @@ -213,11 +253,6 @@ func (s *Service) StreamMediaImage(ctx context.Context, input MediaImageInput) ( traceRecorder.attachToMessage(assistantMessage) } }() - // 媒体任务同样产生用户消息和助手消息，计数语义与普通聊天保持一致。 - if err = s.repo.IncrementMessageCount(ctx, input.ConversationID, 2); err != nil { - retErr = err - return nil, err - } emitMediaEvent(input.OnEvent, "queued", "image task queued") cfg := s.cfg.Snapshot() @@ -230,7 +265,7 @@ func (s *Service) StreamMediaImage(ctx context.Context, input MediaImageInput) ( ConnectTimeoutMS: route.ConnectTimeoutMS, ReadTimeoutMS: route.ReadTimeoutMS, StreamIdleTimeoutMS: route.StreamIdleTimeoutMS, - Endpoint: llm.EndpointImageGenerations, + Endpoint: endpoint, UpstreamModel: route.UpstreamModel, AttributionReferer: attributionReferer, AttributionTitle: attributionTitle, @@ -242,7 +277,7 @@ func (s *Service) StreamMediaImage(ctx context.Context, input MediaImageInput) ( NativeToolAllowedTypesJSON: cfg.NativeToolAllowedTypes, }) - emitMediaEvent(input.OnEvent, "running", "generating image") + emitMediaEvent(input.OnEvent, "running", mediaImageRunningMessage(input.TaskType)) generateInput := llm.GenerateInput{ RequestID: strings.TrimSpace(input.RequestID), ConversationID: input.ConversationID, @@ -252,6 +287,19 @@ func (s *Service) StreamMediaImage(ctx context.Context, input MediaImageInput) ( }}, Options: filteredOptions, } + if input.TaskType == MediaImageTaskEdit { + parts := make([]llm.ContentPart, 0, 1+len(imageEditParts)) + parts = append(parts, llm.ContentPart{ + Kind: llm.ContentPartText, + Text: strings.TrimSpace(input.Prompt), + }) + parts = append(parts, imageEditParts...) + generateInput.Messages = []llm.Message{{ + Role: "user", + Parts: parts, + }} + generateInput.ImageEditMask = maskPart + } var output *llm.GenerateOutput if llm.SupportsImageGenerationStream(routeConfig.Protocol, routeConfig.UpstreamModel) { output, err = s.llmClient.GenerateStream(ctx, routeConfig, generateInput, func(event llm.GenerateStreamEvent) error { @@ -329,32 +377,32 @@ func (s *Service) StreamMediaImage(ctx context.Context, input MediaImageInput) ( UploadedAt: now, }) } - if err = s.repo.CreateAttachments(ctx, attachmentRows); err != nil { - retErr = err - return nil, err - } - usage := output.Usage userMessage.InputTokens = usage.InputTokens userMessage.CacheReadTokens = usage.CacheReadTokens userMessage.CacheWriteTokens = usage.CacheWriteTokens userMessage.TokenUsage = usage.InputTokens + usage.CacheReadTokens + usage.CacheWriteTokens - if err = s.repo.UpdateMessageUsage( - ctx, - userMessage.ID, - usage.InputTokens, - 0, - usage.CacheReadTokens, - usage.CacheWriteTokens, - 0, - ); err != nil { - retErr = err - return nil, err - } content := generatedImageMarkdown(uploaded) latencyMS := time.Since(startedAt).Milliseconds() - if err = s.repo.UpdateAssistantMessageCompletion(ctx, assistantMessage.ID, content, usage.OutputTokens, usage.ReasoningTokens, latencyMS, "success", "", ""); err != nil { + // 上游与文件上传已完成后，数据库侧的附件、用量和完成态仍需保持原子一致。 + if err = s.repo.CompleteAssistantMessageWithAttachments(ctx, + userMessage.ID, + repository.MessageUsageUpdate{ + InputTokens: usage.InputTokens, + CacheReadTokens: usage.CacheReadTokens, + CacheWriteTokens: usage.CacheWriteTokens, + }, + assistantMessage.ID, + repository.AssistantMessageCompletionUpdate{ + Content: content, + OutputTokens: usage.OutputTokens, + ReasoningTokens: usage.ReasoningTokens, + LatencyMS: latencyMS, + Status: "success", + }, + attachmentRows, + ); err != nil { retErr = err return nil, err } @@ -391,6 +439,95 @@ func (s *Service) StreamMediaImage(ctx context.Context, input MediaImageInput) ( }, nil } +func mediaImageUserContentType(taskType MediaImageTaskType) string { + if taskType == MediaImageTaskEdit { + return "mixed" + } + return "text" +} + +func mediaImageRunningMessage(taskType MediaImageTaskType) string { + if taskType == MediaImageTaskEdit { + return "editing image" + } + return "generating image" +} + +// resolveMediaImageEditInputs 读取图片编辑输入图，确保只有图片文件进入图片编辑协议。 +func (s *Service) resolveMediaImageEditInputs(ctx context.Context, input MediaImageInput) ([]AttachmentInput, []llm.ContentPart, error) { + if input.TaskType != MediaImageTaskEdit { + return nil, nil, nil + } + attachments, err := s.resolveAttachments(ctx, input.UserID, input.FileIDs) + if err != nil { + return nil, nil, err + } + if len(attachments) == 0 || len(attachments) > maxMediaImageEditInputImages { + if len(attachments) == 0 { + return nil, nil, ErrMediaImageEditInputRequired + } + return nil, nil, ErrMediaImageEditTooManyInputs + } + parts := make([]llm.ContentPart, 0, len(attachments)) + for _, attachment := range attachments { + if normalizeAttachmentKind(attachment.Kind, attachment.MimeType) != "image" { + return nil, nil, ErrMediaImageEditInputInvalid + } + part, readErr := s.readMediaImageEditFile(ctx, input.UserID, attachment.FileID) + if readErr != nil { + return nil, nil, readErr + } + part.FileName = strings.TrimSpace(attachment.FileName) + parts = append(parts, part) + } + return attachments, parts, nil +} + +func (s *Service) resolveMediaImageEditMask(ctx context.Context, userID uint, fileID string) (*llm.ContentPart, error) { + if strings.TrimSpace(fileID) == "" { + return nil, nil + } + part, err := s.readMediaImageEditFile(ctx, userID, fileID) + if err != nil { + return nil, err + } + return &part, nil +} + +func (s *Service) readMediaImageEditFile(ctx context.Context, userID uint, fileID string) (llm.ContentPart, error) { + content, err := s.OpenFileContent(ctx, userID, strings.TrimSpace(fileID)) + if err != nil { + return llm.ContentPart{}, err + } + defer content.Reader.Close() //nolint:errcheck + + limit := s.cfg.Snapshot().MaxUploadFileBytes + if limit <= 0 { + limit = 20 * 1024 * 1024 + } + data, err := io.ReadAll(io.LimitReader(content.Reader, limit+1)) + if err != nil { + return llm.ContentPart{}, err + } + if int64(len(data)) > limit { + return llm.ContentPart{}, ErrFileTooLarge + } + mimeType := strings.TrimSpace(content.ContentType) + if mimeType == "" { + mimeType = strings.TrimSpace(content.File.DetectedMIME) + } + data, mimeType, err = validateGeneratedImageBytes(data, mimeType) + if err != nil { + return llm.ContentPart{}, ErrMediaImageEditInputInvalid + } + return llm.ContentPart{ + Kind: llm.ContentPartImage, + MimeType: mimeType, + Data: data, + FileName: strings.TrimSpace(content.File.FileName), + }, nil +} + // emitMediaEvent 输出媒体任务状态事件；失败不影响主流程。 func emitMediaEvent(onEvent func(string, map[string]interface{}) error, status string, message string) { if onEvent == nil { diff --git a/backend/internal/application/conversation/service_message_send.go b/backend/internal/application/conversation/service_message_send.go index 0c51682d..9c34b6c0 100644 --- a/backend/internal/application/conversation/service_message_send.go +++ b/backend/internal/application/conversation/service_message_send.go @@ -228,19 +228,11 @@ func (s *Service) sendMessageInternal( ErrorMessage: "", Attachments: string(attachmentsJSON), } - if err = s.repo.CreateMessage(ctx, userMessage); err != nil { - retErr = err - return nil, err - } - userMessage.ParentPublicID = branchState.ParentPublicID - userMessage.SourcePublicID = branchState.SourcePublicID - attachmentRows := make([]model.Attachment, 0, len(resolvedAttachments)) now := time.Now() for _, item := range resolvedAttachments { attachmentRows = append(attachmentRows, model.Attachment{ ConversationID: input.ConversationID, - MessageID: userMessage.ID, UserID: input.UserID, FileID: strings.TrimSpace(item.FileID), Kind: normalizeAttachmentKind(item.Kind, item.MimeType), @@ -254,16 +246,11 @@ func (s *Service) sendMessageInternal( UploadedAt: now, }) } - if err = s.repo.CreateAttachments(ctx, attachmentRows); err != nil { - retErr = err - return nil, err - } assistantMessage = &model.Message{ ConversationID: input.ConversationID, UserID: input.UserID, PublicID: normalizePublicID(uuid.NewString()), - ParentMessageID: &userMessage.ID, RunID: runID, Role: "assistant", ContentType: "text", @@ -281,16 +268,14 @@ func (s *Service) sendMessageInternal( ErrorMessage: "", Attachments: "[]", } - if err = s.repo.CreateMessage(ctx, assistantMessage); err != nil { + // 用户消息、助手占位、用户附件与消息计数必须一起提交，避免失败时留下半个回合。 + if err = s.repo.CreateMessagePairWithUserAttachments(ctx, userMessage, assistantMessage, attachmentRows); err != nil { retErr = err return nil, err } + userMessage.ParentPublicID = branchState.ParentPublicID + userMessage.SourcePublicID = branchState.SourcePublicID assistantMessage.ParentPublicID = userMessage.PublicID - // 两条消息同批次创建，合并为一次 +2，减少一个 DB 往返 - if err = s.repo.IncrementMessageCount(ctx, input.ConversationID, 2); err != nil { - retErr = err - return nil, err - } traceRecorder = newMessageTraceRecorder(s, ctx, assistantMessage, input.OnEvent) if s.routeResolver == nil || s.llmClient == nil { diff --git a/backend/internal/application/settings/model_option_policy.go b/backend/internal/application/settings/model_option_policy.go index f0f4fcfa..0849bf71 100644 --- a/backend/internal/application/settings/model_option_policy.go +++ b/backend/internal/application/settings/model_option_policy.go @@ -10,10 +10,12 @@ var validModelOptionProtocolKeys = map[string]struct{}{ "default": {}, "openai_chat_completions": {}, "openai_image_generations": {}, + "openai_image_edits": {}, "openai_responses": {}, "anthropic_messages": {}, "xai_responses": {}, "xai_image": {}, + "xai_image_edits": {}, "gemini_generate_content": {}, "google_image_generation": {}, } diff --git a/backend/internal/infra/config/config.go b/backend/internal/infra/config/config.go index 01784789..59b3e466 100644 --- a/backend/internal/infra/config/config.go +++ b/backend/internal/infra/config/config.go @@ -65,6 +65,18 @@ func DefaultModelOptionAllowedPathsJSON() string { "style", "user" ], + "openai_image_edits": [ + "background", + "input_fidelity", + "n", + "output_compression", + "output_format", + "partial_images", + "quality", + "response_format", + "size", + "user" + ], "google_image_generation": [ "aspect_ratio", "aspectRatio", @@ -94,6 +106,12 @@ func DefaultModelOptionAllowedPathsJSON() string { "resolution", "response_format" ], + "xai_image_edits": [ + "aspect_ratio", + "n", + "resolution", + "response_format" + ], "gemini_generate_content": [ "generationConfig.temperature", "generationConfig.topP", diff --git a/backend/internal/infra/llm/adapter.go b/backend/internal/infra/llm/adapter.go index 9db85904..2c1e8fa5 100644 --- a/backend/internal/infra/llm/adapter.go +++ b/backend/internal/infra/llm/adapter.go @@ -7,7 +7,7 @@ import ( "strings" ) -// 已支持的协议常量。每个协议固定对应一个 HTTP 端点，不再有「两用」模式。 +// 已支持的协议常量。每个协议固定对应一个 HTTP 端点，任务能力由模型类别和路由规则约束。 const ( AdapterOpenAIResponses = "openai_responses" // POST /v1/responses AdapterOpenAIChatCompletions = "openai_chat_completions" // POST /v1/chat/completions @@ -18,6 +18,7 @@ const ( AdapterGoogleImageGeneration = "google_image_generation" // POST /v1beta/models/{model}:generateContent AdapterXAIResponses = "xai_responses" // POST /v1/responses（OpenAI 兼容） AdapterXAIImage = "xai_image" // POST /v1/images/generations + AdapterXAIImageEdits = "xai_image_edits" // POST /v1/images/edits ) var ( @@ -54,7 +55,8 @@ func IsKnownAdapter(raw string) bool { AdapterGoogleGenerateContent, AdapterGoogleImageGeneration, AdapterXAIResponses, - AdapterXAIImage: + AdapterXAIImage, + AdapterXAIImageEdits: return true default: return false @@ -64,8 +66,8 @@ func IsKnownAdapter(raw string) bool { // IsImplementedAdapter 返回协议是否已有可用的传输层实现。 func IsImplementedAdapter(raw string) bool { switch NormalizeAdapter(raw) { - case AdapterOpenAIResponses, AdapterOpenAIChatCompletions, AdapterOpenAIImageGenerations, AdapterXAIResponses, - AdapterAnthropicMessages, AdapterGoogleGenerateContent, AdapterGoogleImageGeneration, AdapterXAIImage: + case AdapterOpenAIResponses, AdapterOpenAIChatCompletions, AdapterOpenAIImageGenerations, AdapterOpenAIImageEdits, AdapterXAIResponses, + AdapterAnthropicMessages, AdapterGoogleGenerateContent, AdapterGoogleImageGeneration, AdapterXAIImage, AdapterXAIImageEdits: return true default: return false @@ -78,6 +80,7 @@ func SupportsStreamingAdapter(raw string) bool { case AdapterOpenAIResponses, AdapterOpenAIChatCompletions, AdapterOpenAIImageGenerations, + AdapterOpenAIImageEdits, AdapterAnthropicMessages, AdapterGoogleGenerateContent, AdapterGoogleImageGeneration, @@ -88,13 +91,15 @@ func SupportsStreamingAdapter(raw string) bool { } } -// SupportsImageGenerationStream 返回图片生成协议和模型是否支持真实上游流式。 +// SupportsImageGenerationStream 返回图片媒体协议和模型是否支持真实上游流式。 func SupportsImageGenerationStream(protocol string, model string) bool { switch NormalizeAdapter(protocol) { case AdapterOpenAIImageGenerations: return openAIImageGenerationModelSupportsStream(model) case AdapterGoogleImageGeneration: return true + case AdapterOpenAIImageEdits: + return openAIImageEditModelSupportsStream(model) default: return false } @@ -110,6 +115,16 @@ func IsImageGenerationAdapter(raw string) bool { } } +// IsImageEditAdapter 返回协议是否属于独立图片编辑链路。 +func IsImageEditAdapter(raw string) bool { + switch NormalizeAdapter(raw) { + case AdapterOpenAIImageEdits, AdapterGoogleImageGeneration, AdapterXAIImageEdits: + return true + default: + return false + } +} + // DefaultEndpointForAdapter 返回协议对应的固定端点标识。 func DefaultEndpointForAdapter(adapter string) string { switch NormalizeAdapter(adapter) { @@ -117,6 +132,8 @@ func DefaultEndpointForAdapter(adapter string) string { return EndpointChatCompletions case AdapterOpenAIImageGenerations, AdapterGoogleImageGeneration, AdapterXAIImage: return EndpointImageGenerations + case AdapterOpenAIImageEdits, AdapterXAIImageEdits: + return EndpointImageEdits default: // openai_responses、xai_responses 及所有未知值均使用 Responses 端点。 return EndpointResponses diff --git a/backend/internal/infra/llm/adapter_test.go b/backend/internal/infra/llm/adapter_test.go index ae89a655..ba7d0ef7 100644 --- a/backend/internal/infra/llm/adapter_test.go +++ b/backend/internal/infra/llm/adapter_test.go @@ -8,6 +8,9 @@ func TestSupportsStreamingAdapter(t *testing.T) { if !SupportsStreamingAdapter(AdapterOpenAIImageGenerations) { t.Fatalf("expected image generations adapter to support upstream streaming") } + if !SupportsStreamingAdapter(AdapterOpenAIImageEdits) { + t.Fatalf("expected image edits adapter to support upstream streaming") + } if !SupportsStreamingAdapter(AdapterOpenAIResponses) { t.Fatalf("expected responses adapter to support streaming") } @@ -26,10 +29,34 @@ func TestSupportsImageGenerationStream(t *testing.T) { if SupportsStreamingAdapter(AdapterXAIImage) { t.Fatalf("expected xAI image adapter to use non-streaming media flow") } + if SupportsStreamingAdapter(AdapterXAIImageEdits) { + t.Fatalf("expected xAI image edits adapter to use non-streaming media flow") + } if SupportsImageGenerationStream(AdapterOpenAIImageGenerations, "dall-e-3") { t.Fatalf("expected DALL-E models to remain non-streaming") } if SupportsImageGenerationStream(AdapterOpenAIResponses, "gpt-image-1") { t.Fatalf("expected non-image protocol to remain non-streaming for image generation") } + if !SupportsImageGenerationStream(AdapterOpenAIImageEdits, "gpt-image-1") { + t.Fatalf("expected gpt-image edits to support image edit streaming") + } +} + +func TestImageAdapterCapabilities(t *testing.T) { + if !IsImageGenerationAdapter(AdapterGoogleImageGeneration) { + t.Fatalf("expected google image protocol to support image generation") + } + if !IsImageEditAdapter(AdapterGoogleImageGeneration) { + t.Fatalf("expected google image protocol to support image editing") + } + if !IsImageGenerationAdapter(AdapterXAIImage) { + t.Fatalf("expected xAI image protocol to support image generation") + } + if IsImageEditAdapter(AdapterXAIImage) { + t.Fatalf("expected xAI image protocol to stay generation-only") + } + if !IsImageEditAdapter(AdapterXAIImageEdits) { + t.Fatalf("expected xAI image edits protocol to support image editing") + } } diff --git a/backend/internal/infra/llm/client.go b/backend/internal/infra/llm/client.go index c0fbb121..175ece31 100644 --- a/backend/internal/infra/llm/client.go +++ b/backend/internal/infra/llm/client.go @@ -25,6 +25,8 @@ const ( EndpointChatCompletions = "chat_completions" // EndpointImageGenerations 表示 OpenAI Images API 生成端点。 EndpointImageGenerations = "image_generations" + // EndpointImageEdits 表示 OpenAI Images API 编辑端点。 + EndpointImageEdits = "image_edits" ) // 超时默认值。 @@ -133,6 +135,8 @@ type GenerateInput struct { // 非空时：仅在 input 中发送本轮新消息，服务端从存储状态续接历史。 // 空串时：退回全量发送模式，适用于所有 adapter。 PreviousResponseID string + // ImageEditMask 仅供图片编辑 adapter 使用，表示透明区域掩码。 + ImageEditMask *ContentPart } // ToolDefinition 是模型可调用工具的统一声明。 @@ -644,8 +648,10 @@ func NewClientWithEnv(env string, ssrfProtectionEnabled bool) *Client { AdapterOpenAIResponses: &openAIResponsesAdapter{client: client}, AdapterOpenAIChatCompletions: &openAIChatCompletionsAdapter{client: client}, AdapterOpenAIImageGenerations: &openAIImageGenerationsAdapter{client: client}, + AdapterOpenAIImageEdits: &openAIImageEditsAdapter{client: client}, AdapterXAIResponses: &xAIResponsesAdapter{client: client}, AdapterXAIImage: &xAIImageAdapter{client: client}, + AdapterXAIImageEdits: &xAIImageEditsAdapter{client: client}, AdapterAnthropicMessages: &anthropicMessagesAdapter{client: client}, AdapterGoogleGenerateContent: &geminiGenerateContentAdapter{client: client}, AdapterGoogleImageGeneration: &geminiImageGenerationAdapter{client: client}, @@ -1386,6 +1392,8 @@ func normalizeEndpoint(raw string) string { return EndpointChatCompletions case EndpointImageGenerations: return EndpointImageGenerations + case EndpointImageEdits: + return EndpointImageEdits default: return EndpointResponses } diff --git a/backend/internal/infra/llm/endpoint_url_test.go b/backend/internal/infra/llm/endpoint_url_test.go index b9dc0859..c5fb8e8e 100644 --- a/backend/internal/infra/llm/endpoint_url_test.go +++ b/backend/internal/infra/llm/endpoint_url_test.go @@ -27,6 +27,12 @@ func TestBuildOpenAICompatibleURLsRespectVersionedBasePath(t *testing.T) { endpoint: EndpointImageGenerations, want: "https://api.openai.com/v1/images/generations", }, + { + name: "openai image edits endpoint", + baseURL: "https://api.openai.com/v1", + endpoint: EndpointImageEdits, + want: "https://api.openai.com/v1/images/edits", + }, { name: "xai v1 base is not duplicated", baseURL: "https://api.x.ai/v1", diff --git a/backend/internal/infra/llm/gemini_images.go b/backend/internal/infra/llm/gemini_images.go index 7fc5ae25..b2fe98dc 100644 --- a/backend/internal/infra/llm/gemini_images.go +++ b/backend/internal/infra/llm/gemini_images.go @@ -3,6 +3,7 @@ package llm import ( "bytes" "context" + "encoding/base64" "encoding/json" "fmt" "net/http" @@ -10,14 +11,14 @@ import ( "time" ) -// geminiImageGenerationAdapter 实现 Google Gemini 图片生成协议。 +// geminiImageGenerationAdapter 实现 Google Gemini 图片生成/编辑协议。 type geminiImageGenerationAdapter struct { client *Client } func (a *geminiImageGenerationAdapter) Name() string { return AdapterGoogleImageGeneration } -// Generate 调用 Gemini generateContent 图片生成能力，返回结构化图片结果。 +// Generate 调用 Gemini generateContent 图片能力，返回结构化图片结果。 func (a *geminiImageGenerationAdapter) Generate(ctx context.Context, route RouteConfig, input GenerateInput) (*GenerateOutput, error) { return a.client.generateGeminiImageGeneration(ctx, route, input) } @@ -147,7 +148,7 @@ func normalizeGeminiImageGenerationModel(model string) string { } } -// buildGeminiImageGenerationRequestBody 只构造图片生成端点需要的 Gemini 请求字段。 +// buildGeminiImageGenerationRequestBody 构造 Gemini 图片生成/编辑请求字段。 func buildGeminiImageGenerationRequestBody(input GenerateInput) (map[string]interface{}, error) { prompt := buildOpenAIImageGenerationPrompt(input.Messages) if strings.TrimSpace(prompt) == "" { @@ -162,16 +163,37 @@ func buildGeminiImageGenerationRequestBody(input GenerateInput) (map[string]inte return map[string]interface{}{ "contents": []map[string]interface{}{ { - "role": "user", - "parts": []map[string]interface{}{ - {"text": strings.TrimSpace(prompt)}, - }, + "role": "user", + "parts": buildGeminiImageGenerationParts(prompt, collectImageInputParts(input.Messages)), }, }, "generationConfig": generationConfig, }, nil } +// buildGeminiImageGenerationParts 按 Google GenerateContent 格式组合文本提示词和编辑输入图。 +func buildGeminiImageGenerationParts(prompt string, images []ContentPart) []map[string]interface{} { + parts := []map[string]interface{}{ + {"text": strings.TrimSpace(prompt)}, + } + for _, image := range images { + if len(image.Data) == 0 { + continue + } + mimeType := strings.TrimSpace(image.MimeType) + if mimeType == "" { + mimeType = "image/jpeg" + } + parts = append(parts, map[string]interface{}{ + "inline_data": map[string]interface{}{ + "mime_type": mimeType, + "data": base64.StdEncoding.EncodeToString(image.Data), + }, + }) + } + return parts +} + // applyGeminiImageGenerationParams 映射 Google 图片生成文档中的 responseFormat.image 参数。 func applyGeminiImageGenerationParams(generationConfig map[string]interface{}, options map[string]interface{}) { if len(options) == 0 { diff --git a/backend/internal/infra/llm/gemini_test.go b/backend/internal/infra/llm/gemini_test.go index cc4e55c6..a68209bc 100644 --- a/backend/internal/infra/llm/gemini_test.go +++ b/backend/internal/infra/llm/gemini_test.go @@ -130,6 +130,36 @@ func TestBuildGeminiImageGenerationRequestBody(t *testing.T) { } } +func TestBuildGeminiImageGenerationRequestBodyIncludesInlineImages(t *testing.T) { + payload, err := buildGeminiImageGenerationRequestBody(GenerateInput{ + Messages: []Message{ + { + Role: "user", + Parts: []ContentPart{ + {Kind: ContentPartText, Text: "Replace the background"}, + {Kind: ContentPartImage, MimeType: "image/png", Data: []byte("source")}, + }, + }, + }, + }) + if err != nil { + t.Fatalf("build gemini image edit request body: %v", err) + } + + contents := payload["contents"].([]map[string]interface{}) + parts := contents[0]["parts"].([]map[string]interface{}) + if len(parts) != 2 { + t.Fatalf("expected text and image parts, got %#v", parts) + } + if parts[0]["text"] != "Replace the background" { + t.Fatalf("expected prompt text part, got %#v", parts[0]) + } + inlineData := asMap(parts[1]["inline_data"]) + if inlineData["mime_type"] != "image/png" || inlineData["data"] != "c291cmNl" { + t.Fatalf("expected inline image data, got %#v", inlineData) + } +} + func TestNormalizeGeminiImageGenerationModelAliases(t *testing.T) { tests := map[string]string{ "nano-banana-2": "gemini-3.1-flash-image-preview", diff --git a/backend/internal/infra/llm/image_parts.go b/backend/internal/infra/llm/image_parts.go new file mode 100644 index 00000000..ca53e855 --- /dev/null +++ b/backend/internal/infra/llm/image_parts.go @@ -0,0 +1,15 @@ +package llm + +// collectImageInputParts 收集消息中可发送给图片编辑类协议的原始图片输入。 +func collectImageInputParts(messages []Message) []ContentPart { + images := make([]ContentPart, 0) + for _, msg := range messages { + for _, part := range msg.Parts { + if part.Kind != ContentPartImage || len(part.Data) == 0 { + continue + } + images = append(images, part) + } + } + return images +} diff --git a/backend/internal/infra/llm/openai.go b/backend/internal/infra/llm/openai.go index 1e96532c..ca9d8901 100644 --- a/backend/internal/infra/llm/openai.go +++ b/backend/internal/infra/llm/openai.go @@ -303,6 +303,8 @@ func buildOpenAIRequestURL(baseURL string, endpoint string) string { return buildVersionedEndpointURL(baseURL, "v1", "/chat/completions") case EndpointImageGenerations: return buildVersionedEndpointURL(baseURL, "v1", "/images/generations") + case EndpointImageEdits: + return buildVersionedEndpointURL(baseURL, "v1", "/images/edits") default: return buildVersionedEndpointURL(baseURL, "v1", "/responses") } diff --git a/backend/internal/infra/llm/openai_chat_completions.go b/backend/internal/infra/llm/openai_chat_completions.go index e4c3c941..646a021d 100644 --- a/backend/internal/infra/llm/openai_chat_completions.go +++ b/backend/internal/infra/llm/openai_chat_completions.go @@ -440,7 +440,7 @@ func parseOpenAICompatibleUsageForAdapter(adapter string, parsed map[string]inte func openAICompatibleOutputIncludesReasoning(adapter string) bool { switch NormalizeAdapter(adapter) { - case AdapterXAIResponses, AdapterXAIImage: + case AdapterXAIResponses, AdapterXAIImage, AdapterXAIImageEdits: return false default: return true diff --git a/backend/internal/infra/llm/openai_images.go b/backend/internal/infra/llm/openai_images.go index 186aa70a..b625de42 100644 --- a/backend/internal/infra/llm/openai_images.go +++ b/backend/internal/infra/llm/openai_images.go @@ -8,7 +8,9 @@ import ( "errors" "fmt" "io" + "mime/multipart" "net/http" + "net/textproto" "strings" "time" ) @@ -42,6 +44,35 @@ func (a *openAIImageGenerationsAdapter) ListModels(ctx context.Context, route Ro return a.client.listModelsOpenAICompatible(ctx, route) } +// openAIImageEditsAdapter 负责 OpenAI Images API 的图片编辑端点。 +type openAIImageEditsAdapter struct { + client *Client +} + +func (a *openAIImageEditsAdapter) Name() string { return AdapterOpenAIImageEdits } + +// Generate 调用 OpenAI 图片编辑接口，返回结构化图片结果。 +func (a *openAIImageEditsAdapter) Generate(ctx context.Context, route RouteConfig, input GenerateInput) (*GenerateOutput, error) { + route.Endpoint = EndpointImageEdits + return a.client.generateOpenAIImageEdits(ctx, route, input) +} + +// GenerateStream 调用 OpenAI 图片编辑流式接口，事件只输出图片增量，不进入聊天 token delta 链路。 +func (a *openAIImageEditsAdapter) GenerateStream( + ctx context.Context, + route RouteConfig, + input GenerateInput, + onEvent func(GenerateStreamEvent) error, +) (*GenerateOutput, error) { + route.Endpoint = EndpointImageEdits + return a.client.generateOpenAIImageEditsStream(ctx, route, input, onEvent) +} + +// ListModels 复用 OpenAI 兼容 models 目录，供渠道校验和展示使用。 +func (a *openAIImageEditsAdapter) ListModels(ctx context.Context, route RouteConfig) ([]ModelItem, error) { + return a.client.listModelsOpenAICompatible(ctx, route) +} + // generateOpenAIImageGenerations 构造并执行 OpenAI 图片生成请求。 func (c *Client) generateOpenAIImageGenerations(ctx context.Context, route RouteConfig, input GenerateInput) (*GenerateOutput, error) { requestURL := buildOpenAIRequestURL(route.BaseURL, EndpointImageGenerations) @@ -86,7 +117,7 @@ func (c *Client) generateOpenAIImageGenerations(ctx context.Context, route Route return nil, parseUpstreamError(resp.StatusCode, body, upstreamDebugSnapshot(req, payload, resp, body)) } - return parseOpenAIImageGenerationOutput(body, modelParamString(input.Options, "output_format")) + return parseOpenAIImageOutput(body, modelParamString(input.Options, "output_format")) } // generateOpenAIImageGenerationsStream 构造并执行 OpenAI 图片生成流式请求。 @@ -155,7 +186,7 @@ func (c *Client) generateOpenAIImageGenerationsStream( if readErr != nil { return nil, readErr } - output, parseErr := parseOpenAIImageGenerationOutput(body, outputFormat) + output, parseErr := parseOpenAIImageOutput(body, outputFormat) if parseErr != nil { return nil, parseErr } @@ -176,7 +207,7 @@ func (c *Client) generateOpenAIImageGenerationsStream( idleTimeout := resolveStreamIdleTimeout(route.StreamIdleTimeoutMS) idleReader := newIdleTimeoutReader(resp.Body, idleTimeout) streamBody := newUpstreamBodyRecorder(idleReader) - if err = consumeOpenAIImageGenerationStream(streamBody, outputFormat, result, onEvent); err != nil { + if err = consumeOpenAIImageStream(streamBody, outputFormat, result, onEvent); err != nil { return nil, attachUpstreamDebug(err, upstreamDebugSnapshot(req, payload, resp, streamErrorBody(streamBody, err))) } return result, nil @@ -186,6 +217,138 @@ func isEventStreamContentType(contentType string) bool { return strings.Contains(strings.ToLower(strings.TrimSpace(contentType)), "text/event-stream") } +// generateOpenAIImageEdits 构造并执行 OpenAI 图片编辑请求。 +func (c *Client) generateOpenAIImageEdits(ctx context.Context, route RouteConfig, input GenerateInput) (*GenerateOutput, error) { + requestURL := buildOpenAIRequestURL(route.BaseURL, EndpointImageEdits) + if requestURL == "" { + return nil, fmt.Errorf("invalid base url") + } + + payload, contentType, debugBody, err := buildOpenAIImageEditMultipartRequest(route.UpstreamModel, input, false) + if err != nil { + return nil, err + } + + requestCtx, cancel := context.WithTimeout(ctx, resolveReadTimeout(route.ReadTimeoutMS)) + defer cancel() + + req, err := http.NewRequestWithContext(requestCtx, http.MethodPost, requestURL, bytes.NewReader(payload)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", contentType) + if apiKey := strings.TrimSpace(route.APIKey); apiKey != "" { + req.Header.Set("Authorization", "Bearer "+apiKey) + } + setOpenRouterAttributionHeaders(req, route) + setAdditionalHeaders(req, route.HeadersJSON) + + resp, err := c.httpClientForRoute(route).Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() //nolint:errcheck + + body, err := readUpstreamBody(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, parseUpstreamError(resp.StatusCode, body, upstreamDebugSnapshot(req, debugBody, resp, body)) + } + + return parseOpenAIImageOutput(body, modelParamString(input.Options, "output_format")) +} + +// generateOpenAIImageEditsStream 构造并执行 OpenAI 图片编辑流式请求。 +func (c *Client) generateOpenAIImageEditsStream( + ctx context.Context, + route RouteConfig, + input GenerateInput, + onEvent func(GenerateStreamEvent) error, +) (*GenerateOutput, error) { + if !SupportsImageGenerationStream(route.Protocol, route.UpstreamModel) { + return nil, fmt.Errorf("%w: %s", ErrUnsupportedStream, AdapterOpenAIImageEdits) + } + requestURL := buildOpenAIRequestURL(route.BaseURL, EndpointImageEdits) + if requestURL == "" { + return nil, fmt.Errorf("invalid base url") + } + + payload, contentType, debugBody, err := buildOpenAIImageEditMultipartRequest(route.UpstreamModel, input, true) + if err != nil { + return nil, err + } + + firstByteCtx, firstByteCancel := context.WithCancel(ctx) + defer firstByteCancel() + + readTimeout := resolveReadTimeout(route.ReadTimeoutMS) + firstByteTimer := time.AfterFunc(readTimeout, func() { + firstByteCancel() + }) + + req, err := http.NewRequestWithContext(firstByteCtx, http.MethodPost, requestURL, bytes.NewReader(payload)) + if err != nil { + firstByteTimer.Stop() + return nil, err + } + req.Header.Set("Content-Type", contentType) + req.Header.Set("Accept", "text/event-stream") + if apiKey := strings.TrimSpace(route.APIKey); apiKey != "" { + req.Header.Set("Authorization", "Bearer "+apiKey) + } + setOpenRouterAttributionHeaders(req, route) + setAdditionalHeaders(req, route.HeadersJSON) + + resp, err := c.httpClientForRoute(route).Do(req) + firstByteTimer.Stop() + if err != nil { + return nil, err + } + defer resp.Body.Close() //nolint:errcheck + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, readErr := readUpstreamBody(resp.Body) + if readErr != nil { + return nil, readErr + } + return nil, parseUpstreamError(resp.StatusCode, body, upstreamDebugSnapshot(req, debugBody, resp, body)) + } + + outputFormat := modelParamString(input.Options, "output_format") + if !isEventStreamContentType(resp.Header.Get("Content-Type")) { + body, readErr := readUpstreamBody(resp.Body) + if readErr != nil { + return nil, readErr + } + output, parseErr := parseOpenAIImageOutput(body, outputFormat) + if parseErr != nil { + return nil, parseErr + } + if output.Usage != (Usage{}) && onEvent != nil { + if err := onEvent(GenerateStreamEvent{Usage: output.Usage}); err != nil { + return nil, err + } + } + return output, nil + } + + result := &GenerateOutput{ + ResponseID: "", + Usage: Usage{}, + ToolCalls: make([]ToolCall, 0), + ServerToolCalls: make([]ToolCall, 0), + } + idleTimeout := resolveStreamIdleTimeout(route.StreamIdleTimeoutMS) + idleReader := newIdleTimeoutReader(resp.Body, idleTimeout) + streamBody := newUpstreamBodyRecorder(idleReader) + if err = consumeOpenAIImageStream(streamBody, outputFormat, result, onEvent); err != nil { + return nil, attachUpstreamDebug(err, upstreamDebugSnapshot(req, debugBody, resp, streamErrorBody(streamBody, err))) + } + return result, nil +} + // buildOpenAIImageGenerationRequestBody 只允许图片端点支持的请求字段进入上游。 func buildOpenAIImageGenerationRequestBody(model string, input GenerateInput) (map[string]interface{}, error) { prompt := buildOpenAIImageGenerationPrompt(input.Messages) @@ -214,6 +377,143 @@ func buildOpenAIImageGenerationStreamRequestBody(model string, input GenerateInp return payload, nil } +// buildOpenAIImageEditMultipartRequest 构造 OpenAI Images Edits 官方 multipart 请求体。 +func buildOpenAIImageEditMultipartRequest(model string, input GenerateInput, stream bool) ([]byte, string, []byte, error) { + prompt := buildOpenAIImageGenerationPrompt(input.Messages) + if strings.TrimSpace(prompt) == "" { + return nil, "", nil, fmt.Errorf("image edit prompt required") + } + images := collectImageInputParts(input.Messages) + if len(images) == 0 { + return nil, "", nil, fmt.Errorf("image edit input image required") + } + if stream && !openAIImageEditModelSupportsStream(model) { + return nil, "", nil, fmt.Errorf("%w: %s", ErrUnsupportedStream, AdapterOpenAIImageEdits) + } + + formFields := map[string]string{ + "model": strings.TrimSpace(model), + "prompt": prompt, + } + applyOpenAIImageEditParams(formFields, strings.TrimSpace(model), input.Options) + if stream { + formFields["stream"] = "true" + applyOpenAIImageEditStreamParams(formFields, input.Options) + } + + var body bytes.Buffer + writer := multipart.NewWriter(&body) + for key, value := range formFields { + if strings.TrimSpace(value) == "" { + continue + } + if err := writer.WriteField(key, value); err != nil { + return nil, "", nil, err + } + } + for index, image := range images { + fileName := image.FileName + if strings.TrimSpace(fileName) == "" { + fileName = fmt.Sprintf("image-%02d%s", index+1, openAIImageFileExtension(image.MimeType)) + } + if err := writeOpenAIMultipartFile(writer, "image[]", fileName, image.MimeType, image.Data); err != nil { + return nil, "", nil, err + } + } + if input.ImageEditMask != nil && len(input.ImageEditMask.Data) > 0 { + mask := *input.ImageEditMask + fileName := mask.FileName + if strings.TrimSpace(fileName) == "" { + fileName = "mask" + openAIImageFileExtension(mask.MimeType) + } + if err := writeOpenAIMultipartFile(writer, "mask", fileName, mask.MimeType, mask.Data); err != nil { + return nil, "", nil, err + } + } + if err := writer.Close(); err != nil { + return nil, "", nil, err + } + debugBody := buildOpenAIImageEditDebugBody(formFields, len(images), input.ImageEditMask != nil && len(input.ImageEditMask.Data) > 0) + return body.Bytes(), writer.FormDataContentType(), debugBody, nil +} + +func applyOpenAIImageEditParams(fields map[string]string, model string, options map[string]interface{}) { + if len(options) == 0 { + return + } + for _, key := range []string{"quality", "size", "user"} { + if value := modelParamString(options, key); value != "" { + fields[key] = value + } + } + if value := modelParamInt(options, "n"); value > 0 { + fields["n"] = fmt.Sprintf("%d", value) + } + if openAIImageEditModelSupportsGPTImageParams(model) { + for _, key := range []string{"background", "input_fidelity", "output_format"} { + if value := modelParamString(options, key); value != "" { + fields[key] = value + } + } + if value := modelParamInt(options, "output_compression"); value > 0 { + fields["output_compression"] = fmt.Sprintf("%d", value) + } + } + if openAIImageGenerationModelSupportsResponseFormat(model) { + if value := modelParamString(options, "response_format"); value != "" { + fields["response_format"] = value + } + } +} + +func applyOpenAIImageEditStreamParams(fields map[string]string, options map[string]interface{}) { + value, ok := modelParamIntValue(options, "partial_images") + if ok { + if value > 0 { + fields["partial_images"] = fmt.Sprintf("%d", value) + } + return + } + fields["partial_images"] = "1" +} + +func writeOpenAIMultipartFile(writer *multipart.Writer, fieldName string, fileName string, mimeType string, data []byte) error { + if len(data) == 0 { + return fmt.Errorf("multipart file %s is empty", fieldName) + } + normalizedMIME := strings.TrimSpace(mimeType) + if normalizedMIME == "" { + normalizedMIME = "image/png" + } + header := make(textproto.MIMEHeader) + header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, escapeMultipartQuote(fieldName), escapeMultipartQuote(fileName))) + header.Set("Content-Type", normalizedMIME) + part, err := writer.CreatePart(header) + if err != nil { + return err + } + _, err = part.Write(data) + return err +} + +func buildOpenAIImageEditDebugBody(fields map[string]string, imageCount int, hasMask bool) []byte { + payload := make(map[string]interface{}, len(fields)+2) + for key, value := range fields { + payload[key] = value + } + payload["image_count"] = imageCount + payload["mask"] = hasMask + raw, err := json.Marshal(payload) + if err != nil { + return []byte(`{"multipart":true}`) + } + return raw +} + +func escapeMultipartQuote(value string) string { + return strings.NewReplacer("\\", "\\\\", `"`, "\\\"").Replace(strings.TrimSpace(value)) +} + // applyOpenAIImageGenerationParams 从 options 中提取官方 Images API 参数。 func applyOpenAIImageGenerationParams(payload map[string]interface{}, model string, options map[string]interface{}) { if len(options) == 0 { @@ -269,6 +569,15 @@ func openAIImageGenerationModelSupportsGPTImageParams(model string) bool { return strings.HasPrefix(strings.ToLower(strings.TrimSpace(model)), "gpt-image-") } +func openAIImageEditModelSupportsStream(model string) bool { + return openAIImageEditModelSupportsGPTImageParams(model) +} + +func openAIImageEditModelSupportsGPTImageParams(model string) bool { + normalized := strings.ToLower(strings.TrimSpace(model)) + return strings.HasPrefix(normalized, "gpt-image-") || normalized == "chatgpt-image-latest" +} + func openAIImageGenerationModelSupportsResponseFormat(model string) bool { switch strings.ToLower(strings.TrimSpace(model)) { case "dall-e-2", "dall-e-3": @@ -318,9 +627,9 @@ func messagePromptText(msg Message) string { return strings.Join(parts, "\n\n") } -// parseOpenAIImageGenerationOutput 解析 OpenAI 图片响应。 +// parseOpenAIImageOutput 解析 OpenAI 图片响应。 // 图片字节只放入 GeneratedImages，避免把 data URL 写入普通文本链路。 -func parseOpenAIImageGenerationOutput(body []byte, outputFormat string) (*GenerateOutput, error) { +func parseOpenAIImageOutput(body []byte, outputFormat string) (*GenerateOutput, error) { parsed := make(map[string]interface{}) if err := json.Unmarshal(body, &parsed); err != nil { return nil, err @@ -330,11 +639,11 @@ func parseOpenAIImageGenerationOutput(body []byte, outputFormat string) (*Genera ServerToolCalls: make([]ToolCall, 0), RawJSON: string(body), } - applyOpenAIImageGenerationCompletedPayload(parsed, outputFormat, result) + applyOpenAIImageCompletedPayload(parsed, outputFormat, result) return result, nil } -func applyOpenAIImageGenerationCompletedPayload(parsed map[string]interface{}, outputFormat string, result *GenerateOutput) { +func applyOpenAIImageCompletedPayload(parsed map[string]interface{}, outputFormat string, result *GenerateOutput) { if result == nil { return } @@ -359,7 +668,7 @@ func applyOpenAIImageGenerationCompletedPayload(parsed map[string]interface{}, o } if len(data) == 0 { if response := asMap(parsed["response"]); len(response) > 0 { - applyOpenAIImageGenerationCompletedPayload(response, outputFormat, result) + applyOpenAIImageCompletedPayload(response, outputFormat, result) return } } @@ -405,7 +714,7 @@ func parseOpenAIImagePayload(payload map[string]interface{}, outputFormat string return GeneratedImage{}, false } -func consumeOpenAIImageGenerationStream( +func consumeOpenAIImageStream( reader io.Reader, outputFormat string, result *GenerateOutput, @@ -474,9 +783,9 @@ func consumeOpenAIImageGenerationStream( switch { case strings.Contains(eventType, "partial_image"): - return emitOpenAIImageGenerationPartial(parsed, outputFormat, onEvent) + return emitOpenAIImagePartial(parsed, outputFormat, onEvent) case strings.Contains(eventType, "completed") || strings.Contains(eventType, "final"): - applyOpenAIImageGenerationCompletedPayload(parsed, outputFormat, result) + applyOpenAIImageCompletedPayload(parsed, outputFormat, result) } return nil } @@ -529,7 +838,7 @@ func consumeOpenAIImageGenerationStream( result.RawJSON += "\n" } result.RawJSON += payloadText - applyOpenAIImageGenerationCompletedPayload(parsed, outputFormat, result) + applyOpenAIImageCompletedPayload(parsed, outputFormat, result) if result.Usage != (Usage{}) && onEvent != nil { if err := onEvent(GenerateStreamEvent{Usage: result.Usage}); err != nil { return err @@ -539,7 +848,7 @@ func consumeOpenAIImageGenerationStream( return nil } -func emitOpenAIImageGenerationPartial( +func emitOpenAIImagePartial( parsed map[string]interface{}, outputFormat string, onEvent func(GenerateStreamEvent) error, @@ -574,3 +883,16 @@ func openAIImageMIMEType(outputFormat string) string { return "image/png" } } + +func openAIImageFileExtension(mimeType string) string { + switch strings.ToLower(strings.TrimSpace(mimeType)) { + case "image/jpeg", "image/jpg": + return ".jpg" + case "image/webp": + return ".webp" + case "image/gif": + return ".gif" + default: + return ".png" + } +} diff --git a/backend/internal/infra/llm/openai_images_test.go b/backend/internal/infra/llm/openai_images_test.go index fb83cba0..42db2bfb 100644 --- a/backend/internal/infra/llm/openai_images_test.go +++ b/backend/internal/infra/llm/openai_images_test.go @@ -1,6 +1,7 @@ package llm import ( + "bytes" "context" "encoding/json" "net/http" @@ -101,8 +102,86 @@ func TestBuildOpenAIImageGenerationRequestBodyDallEParams(t *testing.T) { } } +func TestBuildOpenAIImageEditMultipartRequest(t *testing.T) { + body, contentType, debugBody, err := buildOpenAIImageEditMultipartRequest("gpt-image-1", GenerateInput{ + Messages: []Message{{ + Role: "user", + Parts: []ContentPart{ + {Kind: ContentPartText, Text: "Replace the background"}, + {Kind: ContentPartImage, MimeType: "image/png", FileName: "source.png", Data: []byte("source")}, + }, + }}, + ImageEditMask: &ContentPart{Kind: ContentPartImage, MimeType: "image/png", FileName: "mask.png", Data: []byte("mask")}, + Options: map[string]interface{}{ + "size": "1024x1024", + "quality": "high", + "output_format": "webp", + "output_compression": 80, + "input_fidelity": "high", + "partial_images": 2, + "stream": true, + "prompt": "override", + }, + }, false) + if err != nil { + t.Fatalf("build image edit multipart request: %v", err) + } + req := httptest.NewRequest(http.MethodPost, "/v1/images/edits", bytes.NewReader(body)) + req.Header.Set("Content-Type", contentType) + if err = req.ParseMultipartForm(10 << 20); err != nil { + t.Fatalf("parse multipart body: %v", err) + } + form := req.MultipartForm + if form.Value["model"][0] != "gpt-image-1" || form.Value["prompt"][0] != "Replace the background" { + t.Fatalf("unexpected edit form fields: %#v", form.Value) + } + if form.Value["output_format"][0] != "webp" || form.Value["input_fidelity"][0] != "high" { + t.Fatalf("expected edit params, got %#v", form.Value) + } + if _, ok := form.Value["stream"]; ok { + t.Fatalf("stream must not be passed by non-streaming edit adapter: %#v", form.Value) + } + if _, ok := form.Value["partial_images"]; ok { + t.Fatalf("partial_images must not be passed without upstream image edit streaming: %#v", form.Value) + } + if len(form.File["image[]"]) != 1 || len(form.File["mask"]) != 1 { + t.Fatalf("expected image[] and mask files, got %#v", form.File) + } + var debug map[string]interface{} + if err = json.Unmarshal(debugBody, &debug); err != nil { + t.Fatalf("parse debug body: %v", err) + } + if debug["image_count"] != float64(1) || debug["mask"] != true { + t.Fatalf("expected sanitized debug body, got %#v", debug) + } +} + +func TestBuildOpenAIImageEditStreamMultipartRequest(t *testing.T) { + body, contentType, _, err := buildOpenAIImageEditMultipartRequest("gpt-image-1", GenerateInput{ + Messages: []Message{{ + Role: "user", + Parts: []ContentPart{ + {Kind: ContentPartText, Text: "Replace the background"}, + {Kind: ContentPartImage, MimeType: "image/png", Data: []byte("source")}, + }, + }}, + Options: map[string]interface{}{"partial_images": 2}, + }, true) + if err != nil { + t.Fatalf("build image edit stream multipart request: %v", err) + } + req := httptest.NewRequest(http.MethodPost, "/v1/images/edits", bytes.NewReader(body)) + req.Header.Set("Content-Type", contentType) + if err = req.ParseMultipartForm(10 << 20); err != nil { + t.Fatalf("parse multipart body: %v", err) + } + if req.MultipartForm.Value["stream"][0] != "true" || req.MultipartForm.Value["partial_images"][0] != "2" { + t.Fatalf("expected edit stream fields, got %#v", req.MultipartForm.Value) + } +} + func TestParseOpenAIImageGenerationOutput(t *testing.T) { - output, err := parseOpenAIImageGenerationOutput([]byte(`{ + output, err := parseOpenAIImageOutput([]byte(`{ "created": 1713833628, "data": [ {"url": "https://example.com/a.png", "revised_prompt": "A revised render"}, @@ -139,7 +218,7 @@ func TestParseOpenAIImageGenerationOutput(t *testing.T) { } func TestParseOpenAIImageGenerationOutputDoesNotInventUsage(t *testing.T) { - output, err := parseOpenAIImageGenerationOutput([]byte(`{ + output, err := parseOpenAIImageOutput([]byte(`{ "data": [{"b64_json": "aGVsbG8="}] }`), "png") if err != nil { @@ -255,3 +334,71 @@ func TestOpenAIImageGenerationStreamFallsBackToJSONResponse(t *testing.T) { t.Fatalf("expected usage event from json fallback, got %#v", usageEvents) } } + +func TestOpenAIImageEditStream(t *testing.T) { + var requestValues map[string][]string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/images/edits" { + t.Fatalf("unexpected path: %s", r.URL.Path) + } + if r.Header.Get("Accept") != "text/event-stream" { + t.Fatalf("expected event stream accept header, got %q", r.Header.Get("Accept")) + } + if err := r.ParseMultipartForm(10 << 20); err != nil { + t.Fatalf("parse multipart request: %v", err) + } + requestValues = r.MultipartForm.Value + if len(r.MultipartForm.File["image[]"]) != 1 { + t.Fatalf("expected one edit image, got %#v", r.MultipartForm.File) + } + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte("event: image_edit.partial_image\n")) + _, _ = w.Write([]byte("data: {\"type\":\"image_edit.partial_image\",\"partial_image_index\":1,\"b64_json\":\"cGFydGlhbA==\"}\n\n")) + _, _ = w.Write([]byte("event: image_edit.completed\n")) + _, _ = w.Write([]byte("data: {\"type\":\"image_edit.completed\",\"id\":\"img_edit_1\",\"b64_json\":\"ZmluYWw=\",\"usage\":{\"input_tokens\":22,\"output_tokens\":44}}\n\n")) + })) + defer server.Close() + + client := NewClient() + var partials []GenerateStreamEvent + output, err := client.GenerateStream(context.Background(), RouteConfig{ + Protocol: AdapterOpenAIImageEdits, + BaseURL: server.URL, + UpstreamModel: "gpt-image-1", + }, GenerateInput{ + Messages: []Message{{ + Role: "user", + Parts: []ContentPart{ + {Kind: ContentPartText, Text: "Replace the background"}, + {Kind: ContentPartImage, MimeType: "image/png", Data: []byte("source")}, + }, + }}, + Options: map[string]interface{}{ + "output_format": "webp", + "partial_images": 2, + }, + }, func(event GenerateStreamEvent) error { + if event.GeneratedImage != nil { + partials = append(partials, event) + } + return nil + }) + if err != nil { + t.Fatalf("generate image edit stream: %v", err) + } + if requestValues["stream"][0] != "true" || requestValues["partial_images"][0] != "2" { + t.Fatalf("expected stream request values, got %#v", requestValues) + } + if len(partials) != 1 || partials[0].GeneratedImage == nil || partials[0].GeneratedImage.B64JSON != "cGFydGlhbA==" { + t.Fatalf("expected partial edit image event, got %#v", partials) + } + if len(output.GeneratedImages) != 1 || output.GeneratedImages[0].B64JSON != "ZmluYWw=" { + t.Fatalf("expected final edit image, got %#v", output.GeneratedImages) + } + if output.ResponseID != "img_edit_1" { + t.Fatalf("expected edit response id, got %q", output.ResponseID) + } + if output.Usage.InputTokens != 22 || output.Usage.OutputTokens != 44 { + t.Fatalf("expected edit usage, got %#v", output.Usage) + } +} diff --git a/backend/internal/infra/llm/xai_images.go b/backend/internal/infra/llm/xai_images.go index fa9b97d7..a4629865 100644 --- a/backend/internal/infra/llm/xai_images.go +++ b/backend/internal/infra/llm/xai_images.go @@ -3,6 +3,7 @@ package llm import ( "bytes" "context" + "encoding/base64" "encoding/json" "fmt" "net/http" @@ -18,6 +19,8 @@ func (a *xAIImageAdapter) Name() string { return AdapterXAIImage } // Generate 调用 xAI 图片生成接口，返回结构化图片结果。 func (a *xAIImageAdapter) Generate(ctx context.Context, route RouteConfig, input GenerateInput) (*GenerateOutput, error) { + route.Protocol = AdapterXAIImage + route.Endpoint = EndpointImageGenerations return a.client.generateXAIImage(ctx, route, input) } @@ -37,16 +40,50 @@ func (a *xAIImageAdapter) ListModels(ctx context.Context, route RouteConfig) ([] return a.client.listModelsOpenAICompatible(ctx, route) } +// xAIImageEditsAdapter 实现 xAI 图片编辑协议。 +type xAIImageEditsAdapter struct { + client *Client +} + +func (a *xAIImageEditsAdapter) Name() string { return AdapterXAIImageEdits } + +// Generate 调用 xAI 图片编辑接口，返回结构化图片结果。 +func (a *xAIImageEditsAdapter) Generate(ctx context.Context, route RouteConfig, input GenerateInput) (*GenerateOutput, error) { + route.Protocol = AdapterXAIImageEdits + route.Endpoint = EndpointImageEdits + return a.client.generateXAIImage(ctx, route, input) +} + +// GenerateStream 当前不伪造图片流式；媒体任务会通过非流式调用落库生成结果。 +func (a *xAIImageEditsAdapter) GenerateStream( + ctx context.Context, + route RouteConfig, + input GenerateInput, + onEvent func(GenerateStreamEvent) error, +) (*GenerateOutput, error) { + return nil, fmt.Errorf("%w: %s", ErrUnsupportedStream, AdapterXAIImageEdits) +} + +// ListModels 复用 xAI models 目录，供渠道校验和展示使用。 +func (a *xAIImageEditsAdapter) ListModels(ctx context.Context, route RouteConfig) ([]ModelItem, error) { + route.Protocol = AdapterXAIImageEdits + return a.client.listModelsOpenAICompatible(ctx, route) +} + // generateXAIImage 构造并执行 xAI Images API 请求。 func (c *Client) generateXAIImage(ctx context.Context, route RouteConfig, input GenerateInput) (*GenerateOutput, error) { - route.Protocol = AdapterXAIImage - route.Endpoint = EndpointImageGenerations - requestURL := buildOpenAIRequestURL(route.BaseURL, EndpointImageGenerations) + protocol := NormalizeAdapter(route.Protocol) + if protocol != AdapterXAIImage && protocol != AdapterXAIImageEdits { + protocol = AdapterXAIImage + } + route.Protocol = protocol + endpoint := DefaultEndpointForAdapter(protocol) + requestURL := buildOpenAIRequestURL(route.BaseURL, endpoint) if requestURL == "" { return nil, fmt.Errorf("invalid base url") } - requestBody, err := buildXAIImageRequestBody(route.UpstreamModel, input) + requestBody, debugBody, err := buildXAIImageRequest(route.UpstreamModel, endpoint, input) if err != nil { return nil, err } @@ -54,6 +91,10 @@ func (c *Client) generateXAIImage(ctx context.Context, route RouteConfig, input if err != nil { return nil, err } + debugPayload := payload + if len(debugBody) > 0 { + debugPayload = debugBody + } requestCtx, cancel := context.WithTimeout(ctx, resolveReadTimeout(route.ReadTimeoutMS)) defer cancel() @@ -79,10 +120,19 @@ func (c *Client) generateXAIImage(ctx context.Context, route RouteConfig, input return nil, err } if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return nil, parseUpstreamError(resp.StatusCode, body, upstreamDebugSnapshot(req, payload, resp, body)) + return nil, parseUpstreamError(resp.StatusCode, body, upstreamDebugSnapshot(req, debugPayload, resp, body)) } - return parseXAIImageOutput(body, modelParamString(input.Options, "response_format")) + return parseXAIImageOutput(body, modelParamString(input.Options, "response_format"), protocol) +} + +// buildXAIImageRequest 根据任务端点构造 xAI 图片生成或编辑请求。 +func buildXAIImageRequest(model string, endpoint string, input GenerateInput) (map[string]interface{}, []byte, error) { + if endpoint == EndpointImageEdits { + return buildXAIImageEditRequestBody(model, input) + } + payload, err := buildXAIImageRequestBody(model, input) + return payload, nil, err } // buildXAIImageRequestBody 只允许 xAI 图片生成端点支持的字段进入上游。 @@ -99,6 +149,53 @@ func buildXAIImageRequestBody(model string, input GenerateInput) (map[string]int return payload, nil } +// buildXAIImageEditRequestBody 只允许 xAI 图片编辑端点支持的字段进入上游。 +func buildXAIImageEditRequestBody(model string, input GenerateInput) (map[string]interface{}, []byte, error) { + prompt := buildOpenAIImageGenerationPrompt(input.Messages) + if strings.TrimSpace(prompt) == "" { + return nil, nil, fmt.Errorf("image edit prompt required") + } + images := collectImageInputParts(input.Messages) + if len(images) == 0 { + return nil, nil, fmt.Errorf("image edit input image required") + } + if len(images) > 3 { + return nil, nil, fmt.Errorf("too many image edit input images") + } + imageInputs := make([]map[string]interface{}, 0, len(images)) + for _, image := range images { + imageInputs = append(imageInputs, xAIImageURLPayload(image)) + } + payload := map[string]interface{}{ + "model": strings.TrimSpace(model), + "prompt": prompt, + } + if len(imageInputs) == 1 { + payload["image"] = imageInputs[0] + } else { + payload["image"] = imageInputs + } + applyXAIImageParams(payload, input.Options) + debugBody, _ := json.Marshal(map[string]interface{}{ + "model": payload["model"], + "prompt": payload["prompt"], + "image_count": len(imageInputs), + }) + return payload, debugBody, nil +} + +// xAIImageURLPayload 将内部图片输入转换为 xAI 文档要求的 image_url 对象。 +func xAIImageURLPayload(image ContentPart) map[string]interface{} { + mimeType := strings.TrimSpace(image.MimeType) + if mimeType == "" { + mimeType = "image/jpeg" + } + return map[string]interface{}{ + "url": "data:" + mimeType + ";base64," + base64.StdEncoding.EncodeToString(image.Data), + "type": "image_url", + } +} + // applyXAIImageParams 从 options 中提取 xAI 图片生成官方参数。 func applyXAIImageParams(payload map[string]interface{}, options map[string]interface{}) { if len(options) == 0 { @@ -115,7 +212,7 @@ func applyXAIImageParams(payload map[string]interface{}, options map[string]inte } // parseXAIImageOutput 解析 xAI 图片响应；图片字节只进入 GeneratedImages。 -func parseXAIImageOutput(body []byte, responseFormat string) (*GenerateOutput, error) { +func parseXAIImageOutput(body []byte, responseFormat string, protocol string) (*GenerateOutput, error) { parsed := make(map[string]interface{}) if err := json.Unmarshal(body, &parsed); err != nil { return nil, err @@ -126,7 +223,7 @@ func parseXAIImageOutput(body []byte, responseFormat string) (*GenerateOutput, e ServerToolCalls: make([]ToolCall, 0), RawJSON: string(body), } - if usage := parseOpenAICompatibleUsageForAdapter(AdapterXAIImage, parsed); usage != (Usage{}) { + if usage := parseOpenAICompatibleUsageForAdapter(protocol, parsed); usage != (Usage{}) { result.Usage = usage } data := asSlice(parsed["data"]) diff --git a/backend/internal/infra/llm/xai_images_test.go b/backend/internal/infra/llm/xai_images_test.go index 376005fa..4538ac62 100644 --- a/backend/internal/infra/llm/xai_images_test.go +++ b/backend/internal/infra/llm/xai_images_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strings" "testing" ) @@ -40,6 +41,66 @@ func TestBuildXAIImageRequestBody(t *testing.T) { } } +func TestBuildXAIImageEditRequestBody(t *testing.T) { + payload, debugBody, err := buildXAIImageEditRequestBody("grok-imagine-image-quality", GenerateInput{ + Messages: []Message{ + { + Role: "user", + Parts: []ContentPart{ + {Kind: ContentPartText, Text: "Render this as a pencil sketch"}, + {Kind: ContentPartImage, MimeType: "image/png", Data: []byte("source")}, + }, + }, + }, + Options: map[string]interface{}{ + "aspect_ratio": "1:1", + "resolution": "2k", + }, + }) + if err != nil { + t.Fatalf("build xAI image edit request body: %v", err) + } + if payload["model"] != "grok-imagine-image-quality" || payload["prompt"] != "Render this as a pencil sketch" { + t.Fatalf("unexpected model or prompt: %#v", payload) + } + if payload["aspect_ratio"] != "1:1" || payload["resolution"] != "2k" { + t.Fatalf("expected xAI edit params, got %#v", payload) + } + image := payload["image"].(map[string]interface{}) + if image["type"] != "image_url" { + t.Fatalf("expected image_url type, got %#v", image) + } + if url := image["url"].(string); !strings.HasPrefix(url, "data:image/png;base64,c291cmNl") { + t.Fatalf("expected base64 data uri, got %q", url) + } + if strings.Contains(string(debugBody), "c291cmNl") { + t.Fatalf("debug body must not include source image bytes: %s", string(debugBody)) + } +} + +func TestBuildXAIImageEditRequestBodyAllowsUpToThreeImages(t *testing.T) { + payload, _, err := buildXAIImageEditRequestBody("grok-imagine-image-quality", GenerateInput{ + Messages: []Message{ + { + Role: "user", + Parts: []ContentPart{ + {Kind: ContentPartText, Text: "Combine these"}, + {Kind: ContentPartImage, MimeType: "image/png", Data: []byte("one")}, + {Kind: ContentPartImage, MimeType: "image/jpeg", Data: []byte("two")}, + {Kind: ContentPartImage, MimeType: "image/webp", Data: []byte("three")}, + }, + }, + }, + }) + if err != nil { + t.Fatalf("build xAI multi-image edit request body: %v", err) + } + images := payload["image"].([]map[string]interface{}) + if len(images) != 3 { + t.Fatalf("expected three ordered image inputs, got %#v", images) + } +} + func TestGenerateXAIImageUsesImageEndpoint(t *testing.T) { var requestPath string var requestBody map[string]interface{} @@ -90,6 +151,88 @@ func TestGenerateXAIImageUsesImageEndpoint(t *testing.T) { } } +func TestGenerateXAIImageGenerationAdapterKeepsGenerationEndpoint(t *testing.T) { + var requestPath string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestPath = r.URL.Path + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[{"url":"https://example.com/generated.jpg"}]}`)) + })) + defer server.Close() + + _, err := NewClient().Generate(context.Background(), RouteConfig{ + Protocol: AdapterXAIImage, + Endpoint: EndpointImageEdits, + BaseURL: server.URL + "/v1", + UpstreamModel: "grok-imagine-image-quality", + }, GenerateInput{ + Messages: []Message{{Role: "user", Content: "A clean product render"}}, + }) + if err != nil { + t.Fatalf("generate xAI image: %v", err) + } + if requestPath != "/v1/images/generations" { + t.Fatalf("expected xAI image generation endpoint, got %q", requestPath) + } +} + +func TestGenerateXAIImageEditUsesImageEditsEndpoint(t *testing.T) { + var requestPath string + var requestBody map[string]interface{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestPath = r.URL.Path + if got := r.Header.Get("Authorization"); got != "Bearer xai-key" { + t.Fatalf("unexpected auth header %q", got) + } + if err := json.NewDecoder(r.Body).Decode(&requestBody); err != nil { + t.Fatalf("decode request body: %v", err) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "id": "img_xai_edit_1", + "data": [ + {"b64_json": "ZWRpdGVk", "revised_prompt": "Edited prompt"} + ], + "usage": {"input_tokens": 13, "output_tokens": 2} + }`)) + })) + defer server.Close() + + output, err := NewClient().Generate(context.Background(), RouteConfig{ + Protocol: AdapterXAIImageEdits, + BaseURL: server.URL + "/v1", + APIKey: "xai-key", + UpstreamModel: "grok-imagine-image-quality", + }, GenerateInput{ + Messages: []Message{{ + Role: "user", + Parts: []ContentPart{ + {Kind: ContentPartText, Text: "Render this as a pencil sketch"}, + {Kind: ContentPartImage, MimeType: "image/png", Data: []byte("source")}, + }, + }}, + Options: map[string]interface{}{ + "response_format": "b64_json", + }, + }) + if err != nil { + t.Fatalf("generate xAI image edit: %v", err) + } + if requestPath != "/v1/images/edits" { + t.Fatalf("expected xAI image edits endpoint, got %q", requestPath) + } + image := asMap(requestBody["image"]) + if image["type"] != "image_url" || !strings.HasPrefix(getString(image["url"]), "data:image/png;base64,c291cmNl") { + t.Fatalf("unexpected edit image payload: %#v", requestBody) + } + if output.ResponseID != "img_xai_edit_1" || len(output.GeneratedImages) != 1 || output.GeneratedImages[0].B64JSON != "ZWRpdGVk" { + t.Fatalf("unexpected edited image output: %#v", output) + } + if output.Usage.InputTokens != 13 || output.Usage.OutputTokens != 2 { + t.Fatalf("expected upstream usage, got %#v", output.Usage) + } +} + func TestParseXAIImageOutput(t *testing.T) { output, err := parseXAIImageOutput([]byte(`{ "id": "img_xai_1", @@ -97,7 +240,7 @@ func TestParseXAIImageOutput(t *testing.T) { {"url": "https://example.com/a.jpg"}, {"b64_json": "aGVsbG8=", "revised_prompt": "A revised render"} ] - }`), "b64_json") + }`), "b64_json", AdapterXAIImage) if err != nil { t.Fatalf("parse xAI image output: %v", err) } diff --git a/backend/internal/infra/persistence/models/channel.go b/backend/internal/infra/persistence/models/channel.go index c0383be4..fd031c0d 100644 --- a/backend/internal/infra/persistence/models/channel.go +++ b/backend/internal/infra/persistence/models/channel.go @@ -74,7 +74,7 @@ type LLMPlatformModelRoute struct { ControlPlaneModel PlatformModelID uint `gorm:"not null;default:0;index:idx_llm_model_routes_model;uniqueIndex:idx_llm_model_routes_unique;comment:平台模型ID"` UpstreamModelID uint `gorm:"not null;default:0;index:idx_llm_model_routes_upstream_model;uniqueIndex:idx_llm_model_routes_unique;comment:上游模型ID"` - Protocol string `gorm:"size:64;not null;index:idx_llm_model_routes_protocol;comment:最终适配器协议"` + Protocol string `gorm:"size:64;not null;index:idx_llm_model_routes_protocol;uniqueIndex:idx_llm_model_routes_unique;comment:最终适配器协议"` Status string `gorm:"size:32;not null;default:'active';index:idx_llm_model_routes_status;comment:路由状态"` Priority int `gorm:"not null;default:1;index:idx_llm_model_routes_priority;comment:路由优先级"` Weight int `gorm:"not null;default:1;comment:负载均衡权重"` diff --git a/backend/internal/infra/persistence/postgres/channel/repository.go b/backend/internal/infra/persistence/postgres/channel/repository.go index 217ae2f2..9f61ae8c 100644 --- a/backend/internal/infra/persistence/postgres/channel/repository.go +++ b/backend/internal/infra/persistence/postgres/channel/repository.go @@ -15,12 +15,34 @@ import ( // channel 包内部的语义错误（ErrUpstreamNotFound 等）优先在调用点直接返回， // 此函数处理未在调用点明确转换的 gorm 错误。 func translateError(err error) error { + if err == nil { + return nil + } if errors.Is(err, gorm.ErrRecordNotFound) { return repository.ErrNotFound } + if isUniqueConstraintError(err) { + return repository.ErrDuplicate + } return err } +type sqlStateError interface { + SQLState() string +} + +func isUniqueConstraintError(err error) bool { + if errors.Is(err, gorm.ErrDuplicatedKey) { + return true + } + var stateErr sqlStateError + if errors.As(err, &stateErr) && stateErr.SQLState() == "23505" { + return true + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "duplicate key") || strings.Contains(msg, "unique constraint") +} + // Repo 封装上游域数据访问。 type Repo struct { db *gorm.DB @@ -684,6 +706,95 @@ func (r *Repo) ListUpstreamModels(ctx context.Context, upstreamID uint, input re return items, total, nil } +// ListUpstreamModelsByNames 按远端模型名集合查询已有上游模型和绑定快照。 +func (r *Repo) ListUpstreamModelsByNames(ctx context.Context, upstreamID uint, upstreamModelNames []string) ([]UpstreamModelListRow, error) { + names := make([]string, 0, len(upstreamModelNames)) + seen := make(map[string]struct{}, len(upstreamModelNames)) + for _, raw := range upstreamModelNames { + name := strings.TrimSpace(raw) + if name == "" { + continue + } + if _, ok := seen[name]; ok { + continue + } + seen[name] = struct{}{} + names = append(names, name) + } + if len(names) == 0 { + return []UpstreamModelListRow{}, nil + } + items := make([]UpstreamModelListRow, 0) + if err := r.db.WithContext(ctx). + Table("llm_upstream_models AS um"). + Select( + "um.*, r.id AS route_id, r.platform_model_id, pm.name AS platform_model_name, pm.vendor AS model_vendor, pm.kinds_json AS model_kinds_json, pm.icon AS model_icon, "+ + "r.protocol, r.status AS route_status, r.priority, r.weight, r.source AS route_source, "+ + "r.cb_failure_threshold, r.cb_duration_min, r.cb_window_min, r.headers_json", + ). + Joins("LEFT JOIN llm_model_routes r ON r.upstream_model_id = um.id"). + Joins("LEFT JOIN llm_platform_models pm ON pm.id = r.platform_model_id"). + Where("um.upstream_id = ? AND um.upstream_model_name IN ?", upstreamID, names). + Order("um.upstream_model_name ASC, r.id ASC NULLS LAST"). + Scan(&items).Error; err != nil { + return nil, translateError(err) + } + return items, nil +} + +// GetUpstreamModelRouteByID 按路由 ID 精确查询上游模型绑定行。 +func (r *Repo) GetUpstreamModelRouteByID(ctx context.Context, upstreamID uint, routeID uint) (*UpstreamModelListRow, error) { + var item UpstreamModelListRow + if err := r.db.WithContext(ctx). + Table("llm_upstream_models AS um"). + Select( + "um.*, r.id AS route_id, r.platform_model_id, pm.name AS platform_model_name, pm.vendor AS model_vendor, pm.kinds_json AS model_kinds_json, pm.icon AS model_icon, "+ + "r.protocol, r.status AS route_status, r.priority, r.weight, r.source AS route_source, "+ + "r.cb_failure_threshold, r.cb_duration_min, r.cb_window_min, r.headers_json", + ). + Joins("JOIN llm_model_routes r ON r.upstream_model_id = um.id"). + Joins("JOIN llm_platform_models pm ON pm.id = r.platform_model_id"). + Where("um.upstream_id = ? AND r.id = ?", upstreamID, routeID). + Take(&item).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, ErrUpstreamModelNotFound + } + return nil, translateError(err) + } + return &item, nil +} + +// GetUpstreamModelRouteByNames 按平台模型、上游模型和协议精确查询绑定行。 +func (r *Repo) GetUpstreamModelRouteByNames( + ctx context.Context, + upstreamID uint, + platformModelName string, + upstreamModelName string, + protocol string, +) (*UpstreamModelListRow, error) { + var item UpstreamModelListRow + query := r.db.WithContext(ctx). + Table("llm_upstream_models AS um"). + Select( + "um.*, r.id AS route_id, r.platform_model_id, pm.name AS platform_model_name, pm.vendor AS model_vendor, pm.kinds_json AS model_kinds_json, pm.icon AS model_icon, "+ + "r.protocol, r.status AS route_status, r.priority, r.weight, r.source AS route_source, "+ + "r.cb_failure_threshold, r.cb_duration_min, r.cb_window_min, r.headers_json", + ). + Joins("JOIN llm_model_routes r ON r.upstream_model_id = um.id"). + Joins("JOIN llm_platform_models pm ON pm.id = r.platform_model_id"). + Where("um.upstream_id = ? AND pm.name = ? AND um.upstream_model_name = ?", upstreamID, strings.TrimSpace(platformModelName), strings.TrimSpace(upstreamModelName)) + if normalizedProtocol := strings.TrimSpace(protocol); normalizedProtocol != "" { + query = query.Where("r.protocol = ?", normalizedProtocol) + } + if err := query.Take(&item).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, ErrUpstreamModelNotFound + } + return nil, translateError(err) + } + return &item, nil +} + func applyUpstreamModelListFilters(query *gorm.DB, input repository.ListChannelUpstreamModelsInput) *gorm.DB { if keyword := strings.TrimSpace(input.Query); keyword != "" { like := "%" + keyword + "%" @@ -737,7 +848,12 @@ func (r *Repo) UpsertPlatformModelRoute(ctx context.Context, item *domainchannel entity := toPlatformModelRouteModel(item) var existing model.LLMPlatformModelRoute query := r.db.WithContext(ctx). - Where("platform_model_id = ? AND upstream_model_id = ?", entity.PlatformModelID, entity.UpstreamModelID). + Where( + "platform_model_id = ? AND upstream_model_id = ? AND protocol = ?", + entity.PlatformModelID, + entity.UpstreamModelID, + entity.Protocol, + ). Limit(1). Find(&existing) if query.Error != nil { @@ -771,6 +887,30 @@ func (r *Repo) UpsertPlatformModelRoute(ctx context.Context, item *domainchannel return nil } +// ListPlatformModelRoutesByPair 查询同一平台模型和同一上游真实模型之间的全部协议绑定。 +func (r *Repo) ListPlatformModelRoutesByPair( + ctx context.Context, + upstreamID uint, + platformModelID uint, + upstreamModelID uint, +) ([]domainchannel.PlatformModelRoute, error) { + items := make([]model.LLMPlatformModelRoute, 0) + if err := r.db.WithContext(ctx). + Table("llm_model_routes AS r"). + Select("r.*"). + Joins("JOIN llm_upstream_models um ON um.id = r.upstream_model_id"). + Where("um.upstream_id = ? AND r.platform_model_id = ? AND r.upstream_model_id = ?", upstreamID, platformModelID, upstreamModelID). + Order("r.id ASC"). + Scan(&items).Error; err != nil { + return nil, translateError(err) + } + results := make([]domainchannel.PlatformModelRoute, 0, len(items)) + for _, item := range items { + results = append(results, toPlatformModelRouteDomain(item)) + } + return results, nil +} + func (r *Repo) GetPlatformModelRouteByID(ctx context.Context, routeID uint, upstreamID uint) (*domainchannel.PlatformModelRoute, error) { var item model.LLMPlatformModelRoute if err := r.db.WithContext(ctx). @@ -901,6 +1041,29 @@ func (r *Repo) ListModelUpstreamSources(ctx context.Context, platformModelName s return items, total, nil } +// GetModelUpstreamSourceByRouteID 按平台模型名和路由 ID 精确查询模型来源。 +func (r *Repo) GetModelUpstreamSourceByRouteID(ctx context.Context, platformModelName string, routeID uint) (*ModelSourceRow, error) { + var item ModelSourceRow + if err := r.db.WithContext(ctx). + Table("llm_model_routes AS r"). + Select( + "r.*, um.upstream_id, u.name AS upstream_name, u.base_url AS base_url, "+ + "um.binding_code, um.upstream_model_name, um.vendor AS upstream_model_vendor, um.icon AS upstream_model_icon, "+ + "um.kinds_json AS upstream_model_kinds_json, um.suggested_protocol, um.status AS upstream_model_status", + ). + Joins("JOIN llm_platform_models pm ON pm.id = r.platform_model_id"). + Joins("JOIN llm_upstream_models um ON um.id = r.upstream_model_id"). + Joins("JOIN llm_upstreams u ON u.id = um.upstream_id"). + Where("pm.name = ? AND r.id = ?", strings.TrimSpace(platformModelName), routeID). + Take(&item).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, ErrUpstreamModelNotFound + } + return nil, translateError(err) + } + return &item, nil +} + // routeScanRow 是 ListActiveRoutesByModel 查询的原始扫描结构体。 // 仅限 infra 层内部使用，扫描后映射到 UpstreamRouteRow。 type routeScanRow struct { diff --git a/backend/internal/infra/persistence/postgres/conversation/repository.go b/backend/internal/infra/persistence/postgres/conversation/repository.go index 2b787d09..066255f7 100644 --- a/backend/internal/infra/persistence/postgres/conversation/repository.go +++ b/backend/internal/infra/persistence/postgres/conversation/repository.go @@ -637,6 +637,62 @@ func (r *Repo) CreateMessage(ctx context.Context, item *domainconversation.Messa return nil } +// CreateMessagePairWithUserAttachments 原子创建用户消息、助手占位消息、用户附件并递增会话消息数。 +func (r *Repo) CreateMessagePairWithUserAttachments( + ctx context.Context, + userMessage *domainconversation.Message, + assistantMessage *domainconversation.Message, + userAttachments []domainconversation.Attachment, +) error { + if userMessage == nil || assistantMessage == nil { + return repository.ErrInvalidInput + } + userAttachmentSnapshot := userMessage.Attachments + assistantAttachmentSnapshot := assistantMessage.Attachments + return translateError(r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + userEntity := toMessageModel(userMessage) + if err := tx.Create(&userEntity).Error; err != nil { + return err + } + *userMessage = toMessageDomain(userEntity) + userMessage.Attachments = userAttachmentSnapshot + + if len(userAttachments) > 0 { + entities := make([]models.Attachment, 0, len(userAttachments)) + for i := range userAttachments { + item := userAttachments[i] + item.ConversationID = userMessage.ConversationID + item.MessageID = userMessage.ID + item.UserID = userMessage.UserID + entities = append(entities, toAttachmentModel(&item)) + } + if err := tx.Create(&entities).Error; err != nil { + return err + } + } + + parentMessageID := userMessage.ID + assistantMessage.ParentMessageID = &parentMessageID + assistantEntity := toMessageModel(assistantMessage) + if err := tx.Create(&assistantEntity).Error; err != nil { + return err + } + *assistantMessage = toMessageDomain(assistantEntity) + assistantMessage.Attachments = assistantAttachmentSnapshot + + result := tx.Model(&models.Conversation{}). + Where("id = ?", userMessage.ConversationID). + Update("message_count", gorm.Expr("message_count + ?", 2)) + if result.Error != nil { + return result.Error + } + if result.RowsAffected == 0 { + return repository.ErrNotFound + } + return nil + })) +} + // GetMessageByPublicID 查询归属会话的消息。 func (r *Repo) GetMessageByPublicID( ctx context.Context, @@ -780,6 +836,68 @@ func (r *Repo) UpdateAssistantMessageCompletion( Error) } +// CompleteAssistantMessageWithAttachments 原子写入助手附件，并同步用户用量与助手完成态。 +func (r *Repo) CompleteAssistantMessageWithAttachments( + ctx context.Context, + userMessageID uint, + userUsage repository.MessageUsageUpdate, + assistantMessageID uint, + assistantCompletion repository.AssistantMessageCompletionUpdate, + assistantAttachments []domainconversation.Attachment, +) error { + return translateError(r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if len(assistantAttachments) > 0 { + entities := make([]models.Attachment, 0, len(assistantAttachments)) + for i := range assistantAttachments { + item := assistantAttachments[i] + item.MessageID = assistantMessageID + entities = append(entities, toAttachmentModel(&item)) + } + if err := tx.Create(&entities).Error; err != nil { + return err + } + } + + userTokenUsage := userUsage.InputTokens + userUsage.CacheReadTokens + userUsage.CacheWriteTokens + userUsage.OutputTokens + userUsage.ReasoningTokens + if userTokenUsage < 0 { + userTokenUsage = 0 + } + if err := tx.Model(&models.Message{}). + Where("id = ?", userMessageID). + Updates(map[string]interface{}{ + "token_usage": userTokenUsage, + "input_tokens": userUsage.InputTokens, + "output_tokens": userUsage.OutputTokens, + "cache_read_tokens": userUsage.CacheReadTokens, + "cache_write_tokens": userUsage.CacheWriteTokens, + "reasoning_tokens": userUsage.ReasoningTokens, + }).Error; err != nil { + return err + } + + assistantTokenUsage := assistantCompletion.OutputTokens + assistantCompletion.ReasoningTokens + if assistantTokenUsage < 0 { + assistantTokenUsage = 0 + } + latencyMS := assistantCompletion.LatencyMS + if latencyMS < 0 { + latencyMS = 0 + } + return tx.Model(&models.Message{}). + Where("id = ?", assistantMessageID). + Updates(map[string]interface{}{ + "content": assistantCompletion.Content, + "token_usage": assistantTokenUsage, + "output_tokens": assistantCompletion.OutputTokens, + "reasoning_tokens": assistantCompletion.ReasoningTokens, + "latency_ms": latencyMS, + "status": assistantCompletion.Status, + "error_code": assistantCompletion.ErrorCode, + "error_message": assistantCompletion.ErrorMessage, + }).Error + })) +} + // UpdateMessageBilling 回填消息计费金额与计费快照。 func (r *Repo) UpdateMessageBilling(ctx context.Context, messageID uint, billedCurrency string, billedNanousd int64, pricingSnapshot string) error { if billedNanousd < 0 { diff --git a/backend/internal/infra/persistence/postgres/postgres.go b/backend/internal/infra/persistence/postgres/postgres.go index 511c0b32..fc0986f9 100644 --- a/backend/internal/infra/persistence/postgres/postgres.go +++ b/backend/internal/infra/persistence/postgres/postgres.go @@ -235,14 +235,21 @@ func escapeSQLLiteral(input string) string { func applyLLMBaselineIndexes(db *gorm.DB) error { statements := []string{ + `ALTER TABLE "llm_upstreams" + ADD COLUMN IF NOT EXISTS "protocol_defaults_json" text NOT NULL DEFAULT '{}'`, + `COMMENT ON COLUMN "llm_upstreams"."protocol_defaults_json" IS '按模型类型配置的默认协议JSON'`, + `ALTER TABLE "llm_platform_models" + ADD COLUMN IF NOT EXISTS "system_prompt" text NOT NULL DEFAULT ''`, + `COMMENT ON COLUMN "llm_platform_models"."system_prompt" IS '模型级系统提示词'`, `CREATE UNIQUE INDEX IF NOT EXISTS idx_llm_upstream_models_upstream_name ON "llm_upstream_models" ("upstream_id", "upstream_model_name")`, `CREATE UNIQUE INDEX IF NOT EXISTS idx_llm_upstream_models_binding_code ON "llm_upstream_models" ("binding_code")`, `CREATE UNIQUE INDEX IF NOT EXISTS idx_llm_platform_models_name ON "llm_platform_models" ("name")`, + `DROP INDEX IF EXISTS idx_llm_model_routes_unique`, `CREATE UNIQUE INDEX IF NOT EXISTS idx_llm_model_routes_unique - ON "llm_model_routes" ("platform_model_id", "upstream_model_id")`, + ON "llm_model_routes" ("platform_model_id", "upstream_model_id", "protocol")`, `CREATE INDEX IF NOT EXISTS idx_llm_model_routes_routing ON "llm_model_routes" ("platform_model_id", "status", "priority", "weight") WHERE status = 'active'`, diff --git a/backend/internal/repository/channel.go b/backend/internal/repository/channel.go index b0c1dd38..733cb658 100644 --- a/backend/internal/repository/channel.go +++ b/backend/internal/repository/channel.go @@ -340,7 +340,12 @@ type ChannelRepository interface { DeleteUpstreamModel(ctx context.Context, sourceID uint, upstreamID uint) error MarkMissingSyncedUpstreamModelsInactive(ctx context.Context, upstreamID uint, activeNames []string) (int64, error) ListUpstreamModels(ctx context.Context, upstreamID uint, input ListChannelUpstreamModelsInput) ([]ChannelUpstreamModelListRow, int64, error) + ListUpstreamModelsByNames(ctx context.Context, upstreamID uint, upstreamModelNames []string) ([]ChannelUpstreamModelListRow, error) + GetUpstreamModelRouteByID(ctx context.Context, upstreamID uint, routeID uint) (*ChannelUpstreamModelListRow, error) + GetUpstreamModelRouteByNames(ctx context.Context, upstreamID uint, platformModelName string, upstreamModelName string, protocol string) (*ChannelUpstreamModelListRow, error) UpsertPlatformModelRoute(ctx context.Context, item *domainchannel.PlatformModelRoute) error + GetModelUpstreamSourceByRouteID(ctx context.Context, platformModelName string, routeID uint) (*ChannelModelSourceRow, error) + ListPlatformModelRoutesByPair(ctx context.Context, upstreamID uint, platformModelID uint, upstreamModelID uint) ([]domainchannel.PlatformModelRoute, error) GetPlatformModelRouteByID(ctx context.Context, routeID uint, upstreamID uint) (*domainchannel.PlatformModelRoute, error) UpdatePlatformModelRouteByID(ctx context.Context, routeID uint, upstreamID uint, input UpdateChannelPlatformRouteInput) error DeletePlatformModelRoute(ctx context.Context, routeID uint, upstreamID uint) error diff --git a/backend/internal/repository/conversation_core.go b/backend/internal/repository/conversation_core.go index 00552fca..4ccbc933 100644 --- a/backend/internal/repository/conversation_core.go +++ b/backend/internal/repository/conversation_core.go @@ -8,6 +8,26 @@ import ( domainuser "github.com/DEEIX-AI/DEEIX-Chat/backend/internal/domain/user" ) +// MessageUsageUpdate 定义消息 token 用量更新字段。 +type MessageUsageUpdate struct { + InputTokens int64 + OutputTokens int64 + CacheReadTokens int64 + CacheWriteTokens int64 + ReasoningTokens int64 +} + +// AssistantMessageCompletionUpdate 定义助手消息完成态更新字段。 +type AssistantMessageCompletionUpdate struct { + Content string + OutputTokens int64 + ReasoningTokens int64 + LatencyMS int64 + Status string + ErrorCode string + ErrorMessage string +} + // ConversationMetadataRepository 封装会话元信息与用户访问能力。 type ConversationMetadataRepository interface { CreateConversation(ctx context.Context, item *domainconversation.Conversation) error @@ -44,6 +64,8 @@ type ConversationMetadataRepository interface { // MessageRepository 封装消息读写能力。 type MessageRepository interface { CreateMessage(ctx context.Context, item *domainconversation.Message) error + CreateMessagePairWithUserAttachments(ctx context.Context, userMessage *domainconversation.Message, assistantMessage *domainconversation.Message, userAttachments []domainconversation.Attachment) error + CompleteAssistantMessageWithAttachments(ctx context.Context, userMessageID uint, userUsage MessageUsageUpdate, assistantMessageID uint, assistantCompletion AssistantMessageCompletionUpdate, assistantAttachments []domainconversation.Attachment) error GetMessageByPublicID(ctx context.Context, conversationID uint, userID uint, publicID string) (*domainconversation.Message, error) GetMessageByPublicIDForUser(ctx context.Context, userID uint, publicID string) (*domainconversation.Message, error) UpdateMessageUsage(ctx context.Context, messageID uint, inputTokens int64, outputTokens int64, cacheReadTokens int64, cacheWriteTokens int64, reasoningTokens int64) error diff --git a/backend/internal/shared/response/error_code.go b/backend/internal/shared/response/error_code.go index 4f5d3e14..e2531f18 100644 --- a/backend/internal/shared/response/error_code.go +++ b/backend/internal/shared/response/error_code.go @@ -145,6 +145,13 @@ var exactErrorSpecs = map[string]errorSpec{ "message generation canceled": {Code: "conversation_run.canceled", Message: "message generation canceled"}, "too many files in one message": {Code: "message.too_many_files", Message: "too many files in one message"}, "generation stream not found": {Code: "conversation_run.stream_not_found", Message: "generation stream not found"}, + "image prompt is required": {Code: "media.image_prompt_required", Message: "image prompt is required"}, + "image generation does not accept input images": {Code: "media.image_generation_rejects_inputs", Message: "image generation does not accept input images"}, + "image edit requires at least one input image": {Code: "media.image_edit_input_required", Message: "image edit requires at least one input image"}, + "too many image edit input images": {Code: "media.image_edit_too_many_inputs", Message: "too many image edit input images"}, + "image edit input image is invalid": {Code: "media.image_edit_input_invalid", Message: "image edit input image is invalid"}, + "media route protocol does not match task": {Code: "media.route_protocol_mismatch", Message: "media route protocol does not match task"}, + "invalid media generation task": {Code: "media.invalid_task", Message: "invalid media generation task"}, "file is required": {Code: "file.required", Message: "file is required"}, "invalid file stream": {Code: "file.invalid_stream", Message: "invalid file stream"}, @@ -175,6 +182,7 @@ var exactErrorSpecs = map[string]errorSpec{ "invalid api keys config": {Code: "llm.invalid_api_keys_config", Message: "invalid api keys config"}, "invalid protocol defaults config": {Code: "llm.invalid_protocol_defaults_config", Message: "invalid protocol defaults config"}, "invalid kinds": {Code: "llm.invalid_kinds", Message: "invalid model kinds"}, + "invalid route protocol combination": {Code: "llm.invalid_route_protocol_combination", Message: "invalid route protocol combination"}, "invalid platform model name": {Code: "llm.invalid_platform_model_name", Message: "invalid platform model name"}, "system prompt too long": {Code: "llm.system_prompt_too_long", Message: "system prompt too long"}, "platform model name is required": {Code: "llm.platform_model_name_required", Message: "platform model name is required"}, @@ -516,6 +524,7 @@ var fallbackMessages = map[string]string{ "llm.invalid_adapter": "invalid adapter", "llm.invalid_compatible": "invalid compatible", "llm.invalid_platform_model_name": "invalid platform model name", + "llm.invalid_route_protocol_combination": "invalid route protocol combination", "llm.system_prompt_too_long": "system prompt too long", "llm.platform_model_name_required": "platform model name is required", "llm.protocol_required": "protocol is required", diff --git a/backend/internal/transport/http/channel/dto_request.go b/backend/internal/transport/http/channel/dto_request.go index 693858cb..16e8f0c6 100644 --- a/backend/internal/transport/http/channel/dto_request.go +++ b/backend/internal/transport/http/channel/dto_request.go @@ -106,10 +106,11 @@ type ImportUpstreamModelsRequest struct { // ImportUpstreamModelItemRequest 单个导入项请求。 type ImportUpstreamModelItemRequest struct { - PlatformModelName string `json:"platformModelName" binding:"required,min=2,max=128"` - UpstreamModelName string `json:"upstreamModelName" binding:"required,min=1,max=128"` - Protocol string `json:"protocol" binding:"omitempty,max=64"` - KindsJSON string `json:"kindsJSON" binding:"omitempty,max=1000"` - Status string `json:"status" binding:"omitempty,oneof=active inactive"` - Priority int `json:"priority"` + PlatformModelName string `json:"platformModelName" binding:"required,min=2,max=128"` + UpstreamModelName string `json:"upstreamModelName" binding:"required,min=1,max=128"` + Protocol string `json:"protocol" binding:"omitempty,max=64"` + Protocols []string `json:"protocols" binding:"omitempty,dive,max=64"` + KindsJSON string `json:"kindsJSON" binding:"omitempty,max=1000"` + Status string `json:"status" binding:"omitempty,oneof=active inactive"` + Priority int `json:"priority"` } diff --git a/backend/internal/transport/http/channel/dto_response.go b/backend/internal/transport/http/channel/dto_response.go index fa8f4dfd..9fe80bb0 100644 --- a/backend/internal/transport/http/channel/dto_response.go +++ b/backend/internal/transport/http/channel/dto_response.go @@ -250,6 +250,7 @@ type UpstreamRemoteModelResponse struct { SuggestedPlatformModelName string `json:"suggestedPlatformModelName"` SuggestedKindsJSON string `json:"suggestedKindsJSON"` SuggestedProtocol string `json:"suggestedProtocol"` + SuggestedProtocols []string `json:"suggestedProtocols"` BindingCode string `json:"bindingCode"` BoundPlatformModels []string `json:"boundPlatformModels"` UpstreamModelStatus string `json:"upstreamModelStatus"` @@ -271,6 +272,7 @@ func toUpstreamRemoteModelsResponse(d appchannel.UpstreamRemoteModelsData) Upstr SuggestedPlatformModelName: item.SuggestedPlatformModelName, SuggestedKindsJSON: item.SuggestedKindsJSON, SuggestedProtocol: item.SuggestedProtocol, + SuggestedProtocols: item.SuggestedProtocols, BindingCode: item.BindingCode, BoundPlatformModels: item.BoundPlatformModels, UpstreamModelStatus: item.UpstreamModelStatus, @@ -335,13 +337,16 @@ type ImportUpstreamModelsResponse struct { } type ImportUpstreamModelResultResponse struct { - UpstreamModelName string `json:"upstreamModelName"` - PlatformModelName string `json:"platformModelName"` - BindingCode string `json:"bindingCode"` - Status string `json:"status"` - CreatedRoute bool `json:"createdRoute"` - CreatedPlatform bool `json:"createdPlatform"` - Error string `json:"error,omitempty"` + UpstreamModelName string `json:"upstreamModelName"` + PlatformModelName string `json:"platformModelName"` + BindingCode string `json:"bindingCode"` + Status string `json:"status"` + CreatedRoute bool `json:"createdRoute"` + CreatedRoutes int `json:"createdRoutes"` + ExistingRoutes int `json:"existingRoutes"` + Protocols []string `json:"protocols"` + CreatedPlatform bool `json:"createdPlatform"` + Error string `json:"error,omitempty"` } func toImportUpstreamModelsResponse(d appchannel.ImportUpstreamModelsData) ImportUpstreamModelsResponse { @@ -353,6 +358,9 @@ func toImportUpstreamModelsResponse(d appchannel.ImportUpstreamModelsData) Impor BindingCode: item.BindingCode, Status: item.Status, CreatedRoute: item.CreatedRoute, + CreatedRoutes: item.CreatedRoutes, + ExistingRoutes: item.ExistingRoutes, + Protocols: item.Protocols, CreatedPlatform: item.CreatedPlatform, Error: item.Error, }) diff --git a/backend/internal/transport/http/channel/handler.go b/backend/internal/transport/http/channel/handler.go index 7ecf3ffe..1f11ccb0 100644 --- a/backend/internal/transport/http/channel/handler.go +++ b/backend/internal/transport/http/channel/handler.go @@ -446,6 +446,8 @@ func (h *Handler) UpsertUpstreamModel(c *gin.Context) { response.Error(c, http.StatusBadRequest, "invalid json config") case errors.Is(err, appchannel.ErrInvalidAdapter): response.Error(c, http.StatusBadRequest, "invalid adapter") + case errors.Is(err, appchannel.ErrInvalidRouteProtocolCombination): + response.Error(c, http.StatusBadRequest, "invalid route protocol combination") case errors.Is(err, appchannel.ErrInvalidKinds): response.Error(c, http.StatusBadRequest, "invalid kinds") case errors.Is(err, appchannel.ErrInvalidPlatformModelName): @@ -785,6 +787,7 @@ func (h *Handler) ImportUpstreamModels(c *gin.Context) { PlatformModelName: item.PlatformModelName, UpstreamModelName: item.UpstreamModelName, Protocol: item.Protocol, + Protocols: item.Protocols, KindsJSON: item.KindsJSON, Status: item.Status, Priority: item.Priority, @@ -800,6 +803,8 @@ func (h *Handler) ImportUpstreamModels(c *gin.Context) { response.Error(c, http.StatusNotFound, "upstream not found") case errors.Is(err, appchannel.ErrInvalidAdapter): response.ErrorFrom(c, http.StatusBadRequest, err) + case errors.Is(err, appchannel.ErrInvalidRouteProtocolCombination): + response.ErrorFrom(c, http.StatusBadRequest, err) case errors.Is(err, appchannel.ErrInvalidPlatformModelName): response.ErrorFrom(c, http.StatusBadRequest, err) case errors.Is(err, appchannel.ErrNoActiveKey): @@ -1134,6 +1139,14 @@ func (h *Handler) UpdateModelUpstreamSource(c *gin.Context) { response.Error(c, http.StatusNotFound, "model not found") case errors.Is(err, appchannel.ErrUpstreamModelNotFound): response.Error(c, http.StatusNotFound, "upstream model not found") + case errors.Is(err, appchannel.ErrInvalidAdapter): + response.Error(c, http.StatusBadRequest, "invalid adapter") + case errors.Is(err, appchannel.ErrProtocolRequired): + response.Error(c, http.StatusBadRequest, "protocol required") + case errors.Is(err, appchannel.ErrInvalidRouteProtocolCombination): + response.Error(c, http.StatusBadRequest, "invalid route protocol combination") + case errors.Is(err, appchannel.ErrUpstreamModelConflict): + response.Error(c, http.StatusConflict, "target model already bound on this upstream") default: response.Error(c, http.StatusInternalServerError, "update model upstream source failed") } diff --git a/backend/internal/transport/http/conversation/dto_request.go b/backend/internal/transport/http/conversation/dto_request.go index 6723cd67..9d6f3d25 100644 --- a/backend/internal/transport/http/conversation/dto_request.go +++ b/backend/internal/transport/http/conversation/dto_request.go @@ -97,6 +97,7 @@ type MediaImageRequest struct { Options map[string]interface{} `json:"options"` ClientRunID string `json:"clientRunID" binding:"omitempty,max=64"` FileIDs []string `json:"fileIDs" binding:"max=20"` + MaskFileID string `json:"maskFileID" binding:"omitempty,max=128"` ParentMessagePublicID string `json:"parentMessagePublicID" binding:"omitempty,max=32"` SourceMessagePublicID string `json:"sourceMessagePublicID" binding:"omitempty,max=32"` BranchReason string `json:"branchReason" binding:"omitempty,oneof=default retry edit"` diff --git a/backend/internal/transport/http/conversation/handler.go b/backend/internal/transport/http/conversation/handler.go index e0f1bbe9..f41178c5 100644 --- a/backend/internal/transport/http/conversation/handler.go +++ b/backend/internal/transport/http/conversation/handler.go @@ -129,15 +129,30 @@ func mapStreamError(err error) streamError { case errors.Is(err, appconversation.ErrMessageGenerationCanceled): status = http.StatusBadRequest message = "message generation canceled" + case errors.Is(err, appconversation.ErrMediaImagePromptRequired): + status = http.StatusBadRequest + message = "image prompt is required" + case errors.Is(err, appconversation.ErrMediaImageGenerationRejectsInputs): + status = http.StatusBadRequest + message = "image generation does not accept input images" + case errors.Is(err, appconversation.ErrMediaImageEditInputRequired): + status = http.StatusBadRequest + message = "image edit requires at least one input image" + case errors.Is(err, appconversation.ErrMediaImageEditTooManyInputs): + status = http.StatusBadRequest + message = "too many image edit input images" + case errors.Is(err, appconversation.ErrMediaImageEditInputInvalid): + status = http.StatusBadRequest + message = "image edit input image is invalid" + case errors.Is(err, appconversation.ErrMediaRouteProtocolMismatch): + status = http.StatusServiceUnavailable + message = "media route protocol does not match task" case errors.Is(err, appconversation.ErrInvalidMediaGenerationTask): status = http.StatusBadRequest message = "invalid media generation task" case errors.Is(err, appconversation.ErrDuplicateMessageGenerationRun): status = http.StatusConflict message = "message generation run already exists" - case errors.Is(err, appconversation.ErrMediaImageEditNotImplemented): - status = http.StatusNotImplemented - message = "image edit protocol not implemented" case errors.Is(err, appconversation.ErrUpstreamRequestFailed): status = http.StatusBadGateway message = mapClientErrorMessage(err) diff --git a/backend/internal/transport/http/conversation/handler_media.go b/backend/internal/transport/http/conversation/handler_media.go index 740dd348..7bceeecf 100644 --- a/backend/internal/transport/http/conversation/handler_media.go +++ b/backend/internal/transport/http/conversation/handler_media.go @@ -94,6 +94,7 @@ func (h *Handler) streamMediaImage(c *gin.Context, taskType appconversation.Medi Options: req.Options, ClientRunID: req.ClientRunID, FileIDs: req.FileIDs, + MaskFileID: req.MaskFileID, ParentMessagePublicID: req.ParentMessagePublicID, SourceMessagePublicID: req.SourceMessagePublicID, BranchReason: req.BranchReason, diff --git a/frontend/features/admin/api/llm.types.ts b/frontend/features/admin/api/llm.types.ts index f50e77a4..d21543da 100644 --- a/frontend/features/admin/api/llm.types.ts +++ b/frontend/features/admin/api/llm.types.ts @@ -11,7 +11,8 @@ export type AdminLLMAdapter = | "google_generate_content" | "google_image_generation" | "xai_responses" - | "xai_image"; + | "xai_image" + | "xai_image_edits"; export type AdminLLMModelVendor = string; export type AdminLLMCompatible = | "openai" @@ -141,6 +142,7 @@ export type AdminLLMRemoteModelItem = { suggestedPlatformModelName: string; suggestedKindsJSON: string; suggestedProtocol: AdminLLMAdapter | ""; + suggestedProtocols: AdminLLMAdapter[]; bindingCode: string; boundPlatformModels: string[]; upstreamModelStatus: AdminLLMStatus | ""; @@ -251,6 +253,7 @@ export type ImportAdminLLMUpstreamModelsRequest = { platformModelName: string; upstreamModelName: string; protocol?: AdminLLMAdapter; + protocols?: AdminLLMAdapter[]; kindsJSON?: string; status?: AdminLLMStatus; priority?: number; @@ -305,6 +308,9 @@ export type ImportAdminLLMUpstreamModelsData = { bindingCode: string; status: "created" | "existing" | "failed"; createdRoute: boolean; + createdRoutes: number; + existingRoutes: number; + protocols: AdminLLMAdapter[]; createdPlatform: boolean; error?: string; }>; diff --git a/frontend/features/admin/components/sections/models/models-dialog.tsx b/frontend/features/admin/components/sections/models/models-dialog.tsx index 755080bf..099fc4e0 100644 --- a/frontend/features/admin/components/sections/models/models-dialog.tsx +++ b/frontend/features/admin/components/sections/models/models-dialog.tsx @@ -77,10 +77,7 @@ export function DeleteModelDialog({ {t("deleteDialog.title")} - {t.rich("deleteDialog.description", { - model: target?.platformModelName ?? "", - strong: (chunks) => {chunks}, - })} + {t("deleteDialog.description", { model: target?.platformModelName ?? "" })} diff --git a/frontend/features/admin/components/sections/models/models-table.tsx b/frontend/features/admin/components/sections/models/models-table.tsx index 5930c428..eee6e0cb 100644 --- a/frontend/features/admin/components/sections/models/models-table.tsx +++ b/frontend/features/admin/components/sections/models/models-table.tsx @@ -946,9 +946,8 @@ export function ModelsTable({ {t("sources.deleteTitle")} - {t.rich("sources.deleteDescription", { + {t("sources.deleteDescription", { name: deleteSourceTarget?.source.upstreamModelName ?? "", - strong: (chunks) => {chunks}, })} diff --git a/frontend/features/admin/components/sections/settings/admin-settings-conversation.tsx b/frontend/features/admin/components/sections/settings/admin-settings-conversation.tsx index de10d9d0..d5d66715 100644 --- a/frontend/features/admin/components/sections/settings/admin-settings-conversation.tsx +++ b/frontend/features/admin/components/sections/settings/admin-settings-conversation.tsx @@ -400,6 +400,18 @@ generationConfig.safetySettings.threshold`} "response_format", "style", "user" + ], + "openai_image_edits": [ + "background", + "input_fidelity", + "n", + "output_compression", + "output_format", + "partial_images", + "quality", + "response_format", + "size", + "user" ], "google_image_generation": [ "aspect_ratio", @@ -421,6 +433,12 @@ generationConfig.safetySettings.threshold`} "resolution", "response_format" ], + "xai_image_edits": [ + "aspect_ratio", + "n", + "resolution", + "response_format" + ], "openai_chat_completions": [ "service_tier", "thinking.type" @@ -458,7 +476,7 @@ generationConfig.safetySettings.threshold`}

{t("guide.protocolTitle")}

{t("guide.protocolDescription")}

- {["default", "openai_chat_completions", "openai_responses", "openai_image_generations", "google_image_generation", "xai_image", "anthropic_messages", "xai_responses", "gemini_generate_content"].map((item) => ( + {["default", "openai_chat_completions", "openai_responses", "openai_image_generations", "openai_image_edits", "google_image_generation", "xai_image", "xai_image_edits", "anthropic_messages", "xai_responses", "gemini_generate_content"].map((item) => ( {item} ))}
diff --git a/frontend/features/admin/components/sections/upstreams/upstream-dialog.tsx b/frontend/features/admin/components/sections/upstreams/upstream-dialog.tsx index df6ed6ad..16598e27 100644 --- a/frontend/features/admin/components/sections/upstreams/upstream-dialog.tsx +++ b/frontend/features/admin/components/sections/upstreams/upstream-dialog.tsx @@ -77,9 +77,7 @@ export function DeleteUpstreamDialog({ {t("deleteDialog.title")} - {t.rich("deleteDialog.description", { - name: () => {upstream?.name}, - })} + {t("deleteDialog.description", { name: upstream?.name ?? "" })} diff --git a/frontend/features/admin/components/sections/upstreams/upstream-models-dialog.tsx b/frontend/features/admin/components/sections/upstreams/upstream-models-dialog.tsx index 1e0813d4..8c459f53 100644 --- a/frontend/features/admin/components/sections/upstreams/upstream-models-dialog.tsx +++ b/frontend/features/admin/components/sections/upstreams/upstream-models-dialog.tsx @@ -70,6 +70,8 @@ import type { } from "@/features/admin/api/llm.types"; import { PROTOCOL_OPTIONS, + resolveKindsDisplayForProtocols, + resolveNextRouteProtocolSelection, } from "@/features/admin/utils/llm-display"; import { MODEL_KIND_OPTIONS, PAGE_SIZE_DEFAULT } from "@/features/admin/types/llm"; import { @@ -158,6 +160,82 @@ function KindsDropdown({ ); } +function ProtocolsDropdown({ + value, + onChange, + disabled, + className, +}: { + value: AdminLLMAdapter[]; + onChange: (value: AdminLLMAdapter[]) => void; + disabled?: boolean; + className?: string; +}) { + const t = useTranslations("adminChannels"); + const selected = React.useMemo(() => new Set(value), [value]); + const selectedLabel = React.useMemo( + () => + PROTOCOL_OPTIONS + .filter((item) => selected.has(item.value)) + .map((item) => item.label) + .join(", "), + [selected], + ); + + function toggle(protocol: AdminLLMAdapter) { + onChange(resolveNextRouteProtocolSelection(value, protocol)); + } + + return ( + + + + + + + {PROTOCOL_OPTIONS.map((item) => ( + + ))} + + + ); +} + function BulkActionControlRow({ icon, label, @@ -188,16 +266,25 @@ function BulkActionControlRow({ ); } -// --------------------------------------------------------------------------- -// Local draft type -// --------------------------------------------------------------------------- +function routeIDsForRow(row: RowDraft): number[] { + return Object.values(row.routeIDsByProtocol).filter((id) => id > 0); +} + +function selectedProtocolsForSave(row: RowDraft): AdminLLMAdapter[] { + const protocols = row.protocols.length > 0 ? row.protocols : []; + return Array.from(new Set(protocols)); +} -const AUTO_PROTOCOL_VALUE = "__auto__"; +async function runOperationsInOrder(operations: Array<() => Promise>): Promise { + for (const operation of operations) { + await operation(); + } +} type ModelRowProps = { row: RowDraft; isSelected: boolean; - onSelect: (id: number, checked: boolean) => void; + onSelect: (draftKey: string, checked: boolean) => void; onUpdate: (draftKey: string, patch: Partial>) => void; }; @@ -206,7 +293,7 @@ const ModelRow = React.memo(function ModelRow({ row, isSelected, onSelect, onUpd const platformModelName = row.platformModelNameDraft.trim(); const hasBindingDraft = platformModelName.length > 0; const routeChecked = row.routeStatus === "active"; - const protocolValue = hasBindingDraft ? row.protocol || row.suggestedProtocol || AUTO_PROTOCOL_VALUE : AUTO_PROTOCOL_VALUE; + const persistedRouteCount = routeIDsForRow(row).length; const handlePlatformModelChange = (value: string) => { onUpdate(row.draftKey, { platformModelNameDraft: value }); @@ -223,8 +310,8 @@ const ModelRow = React.memo(function ModelRow({ row, isSelected, onSelect, onUpd
onSelect(row.routeID, checked === true)} + disabled={persistedRouteCount === 0} + onCheckedChange={(checked) => onSelect(row.draftKey, checked === true)} aria-label={t("modelsDialog.selectModel", { name: row.upstreamModelName })} />
@@ -250,32 +337,22 @@ const ModelRow = React.memo(function ModelRow({ row, isSelected, onSelect, onUpd onChange={(e) => handlePlatformModelChange(e.target.value)} /> - + {!hasBindingDraft ? ( {t("modelsDialog.deleteAfterSave")} ) : ( - + /> )} @@ -315,6 +392,10 @@ function dedupeRemoteModels(items: AdminLLMRemoteModelItem[]): AdminLLMRemoteMod suggestedPlatformModelName: existing.suggestedPlatformModelName || item.suggestedPlatformModelName, suggestedKindsJSON: existing.suggestedKindsJSON || item.suggestedKindsJSON, suggestedProtocol: existing.suggestedProtocol || item.suggestedProtocol, + suggestedProtocols: Array.from(new Set([ + ...(existing.suggestedProtocols ?? []), + ...(item.suggestedProtocols ?? []), + ])), bindingCode: existing.bindingCode || item.bindingCode, boundPlatformModels: Array.from(new Set([...existing.boundPlatformModels, ...item.boundPlatformModels])), upstreamModelStatus: existing.upstreamModelStatus || item.upstreamModelStatus, @@ -379,7 +460,7 @@ function RemoteModelsDialog({ setSelected(new Set(syncableItems.map((i) => i.upstreamModelName))); setDraftPlatformModelNames(createDraftPlatformModelNameMap(syncableItems)); } catch (err) { - toast.error(resolveErrorMessage(err, t("modelsDialog.remoteLoadFailed"))); + toast.error(t("modelsDialog.remoteLoadFailed"), { description: resolveErrorMessage(err) }); onOpenChange(false); } finally { setLoading(false); @@ -428,7 +509,7 @@ function RemoteModelsDialog({ .map((i) => ({ upstreamModelName: i.upstreamModelName, platformModelName: (draftPlatformModelNames.get(i.upstreamModelName) || i.upstreamModelName).trim(), - protocol: i.suggestedProtocol || undefined, + protocols: (i.suggestedProtocols?.length ? i.suggestedProtocols : i.suggestedProtocol ? [i.suggestedProtocol] : undefined), kindsJSON: i.suggestedKindsJSON || undefined, })); const result = await importAdminLLMUpstreamModels(token, upstream.id, { items }); @@ -447,7 +528,7 @@ function RemoteModelsDialog({ onImported(); onOpenChange(false); } catch (err) { - toast.error(resolveErrorMessage(err, t("modelsDialog.importFailed"))); + toast.error(t("modelsDialog.importFailed"), { description: resolveErrorMessage(err) }); } finally { setImporting(false); } @@ -461,6 +542,7 @@ function RemoteModelsDialog({ item.upstreamModelName, item.suggestedPlatformModelName || "", item.suggestedProtocol || "", + ...(item.suggestedProtocols ?? []), t(`modelsDialog.remoteStatus.${remoteModelStatusKey(item)}`), ].some((value) => value.toLowerCase().includes(normalizedQuery)); }); @@ -613,19 +695,24 @@ function NewBindingDialog({ const payload: UpsertAdminLLMUpstreamModelRequest = { upstreamModelName: form.upstreamModelName.trim(), platformModelName: form.platformModelName.trim(), - protocol: form.protocol, kindsJSON: displayToKindsJson(form.kindsDisplay), status: form.status, priority: 1, weight: 1, }; - await upsertAdminLLMUpstreamModel(token, upstreamId, payload); + const protocols = form.protocols.length > 0 ? form.protocols : [undefined]; + for (const protocol of protocols) { + await upsertAdminLLMUpstreamModel(token, upstreamId, { + ...payload, + protocol, + }); + } toast.success(t("modelsDialog.bindingCreated")); setForm(DEFAULT_NEW_BINDING); onOpenChange(false); onCreated(); } catch (err) { - toast.error(resolveErrorMessage(err, t("toast.createFailed"))); + toast.error(t("toast.createFailed"), { description: resolveErrorMessage(err) }); } finally { setSaving(false); } @@ -661,21 +748,17 @@ function NewBindingDialog({
- + + setForm((prev) => ({ + ...prev, + protocols, + kindsDisplay: resolveKindsDisplayForProtocols(protocols, prev.kindsDisplay), + })) + } + className="h-9 text-sm" + />
@@ -774,10 +857,10 @@ export function UpstreamModelsDialog({ const [saving, setSaving] = React.useState(false); const [deleting, setDeleting] = React.useState(false); const [deleteConfirmOpen, setDeleteConfirmOpen] = React.useState(false); - const [selected, setSelected] = React.useState>(new Set()); + const [selected, setSelected] = React.useState>(new Set()); const [newBindingOpen, setNewBindingOpen] = React.useState(false); const [bulkRouteStatus, setBulkRouteStatus] = React.useState<"active" | "inactive">("active"); - const [bulkProtocol, setBulkProtocol] = React.useState("openai_responses"); + const [bulkProtocols, setBulkProtocols] = React.useState(["openai_responses"]); const [bulkKindsDisplay, setBulkKindsDisplay] = React.useState("chat"); const [bulkPatchConfirm, setBulkPatchConfirm] = React.useState(null); const [query, setQuery] = React.useState(""); @@ -816,7 +899,7 @@ export function UpstreamModelsDialog({ setRows([]); setTotal(0); setLoadedUpstreamID(upstreamID); - toast.error(resolveErrorMessage(err, t("modelsDialog.loadFailed"))); + toast.error(t("modelsDialog.loadFailed"), { description: resolveErrorMessage(err) }); } finally { if (requestSeq === requestSeqRef.current) { setLoadingList(false); @@ -892,30 +975,32 @@ export function UpstreamModelsDialog({ setListParams((prev) => ({ ...prev, ...patch, page: patch.page ?? 1 })); }, []); - const allSelected = - visibleRows.some((r) => r.routeID > 0) && visibleRows.filter((r) => r.routeID > 0).every((r) => selected.has(r.routeID)); - const someSelected = visibleRows.some((r) => r.routeID > 0 && selected.has(r.routeID)); + const selectableRows = React.useMemo( + () => visibleRows.filter((row) => routeIDsForRow(row).length > 0), + [visibleRows], + ); + const allSelected = selectableRows.length > 0 && selectableRows.every((row) => selected.has(row.draftKey)); + const someSelected = selectableRows.some((row) => selected.has(row.draftKey)); function handleSelectAll(checked: boolean) { setSelected((current) => { const next = new Set(current); - for (const row of visibleRows) { - if (!row.routeID) continue; + for (const row of selectableRows) { if (checked) { - next.add(row.routeID); + next.add(row.draftKey); } else { - next.delete(row.routeID); + next.delete(row.draftKey); } } return next; }); } - const handleSelectOne = React.useCallback((id: number, checked: boolean) => { + const handleSelectOne = React.useCallback((draftKey: string, checked: boolean) => { setSelected((prev) => { const next = new Set(prev); - if (checked) next.add(id); - else next.delete(id); + if (checked) next.add(draftKey); + else next.delete(draftKey); return next; }); }, []); @@ -935,7 +1020,7 @@ export function UpstreamModelsDialog({ if (selected.size === 0) return; setRows((prev) => prev.map((row) => - row.routeID > 0 && selected.has(row.routeID) + routeIDsForRow(row).length > 0 && selected.has(row.draftKey) ? { ...row, ...patch, isDirty: true } : row, ), @@ -944,18 +1029,24 @@ export function UpstreamModelsDialog({ async function handleDeleteSelected() { if (!upstream || selected.size === 0) return; + const routeIDs = rows + .filter((row) => selected.has(row.draftKey)) + .flatMap(routeIDsForRow); + if (routeIDs.length === 0) return; setDeleting(true); try { const token = await resolveAccessToken(); const result = await batchDeleteAdminLLMUpstreamModels(token, upstream.id, { - ids: Array.from(selected), + ids: routeIDs, }); const deletedIDs = new Set( result.results .filter((item) => item.status === "deleted" || item.status === "not_found") .map((item) => item.id), ); - setRows((prev) => prev.filter((row) => !deletedIDs.has(row.routeID))); + setRows((prev) => + prev.filter((row) => routeIDsForRow(row).some((routeID) => !deletedIDs.has(routeID))), + ); setSelected(new Set()); if (result.failedCount > 0) { toast.error(t("modelsDialog.batchDeletePartialFailed"), { @@ -975,7 +1066,7 @@ export function UpstreamModelsDialog({ void loadBindings(); onUpstreamUpdated({ ...upstream }); } catch (err) { - toast.error(resolveErrorMessage(err, t("toast.deleteFailed"))); + toast.error(t("toast.deleteFailed"), { description: resolveErrorMessage(err) }); } finally { setDeleting(false); setDeleteConfirmOpen(false); @@ -1002,47 +1093,92 @@ export function UpstreamModelsDialog({ setSaving(true); try { const token = await resolveAccessToken(); - const operations: Array> = []; + const deleteOperations: Array<() => Promise> = []; + const upsertOperations: Array<() => Promise> = []; let savedCount = 0; let deletedCount = 0; for (const row of dirty) { const platformModelName = row.platformModelNameDraft.trim(); + const existingRouteIDs = routeIDsForRow(row); const shouldDeleteRoute = - row.routeID > 0 && + existingRouteIDs.length > 0 && row.routeStatus === "inactive" && platformModelName.length === 0; if (shouldDeleteRoute) { - operations.push(deleteAdminLLMUpstreamModel(token, upstream.id, row.routeID)); - deletedCount += 1; + for (const routeID of existingRouteIDs) { + deleteOperations.push(() => deleteAdminLLMUpstreamModel(token, upstream.id, routeID)); + deletedCount += 1; + } continue; } if (!platformModelName) { continue; } - const payload: UpsertAdminLLMUpstreamModelRequest = { - routeID: row.routeID, + const basePayload: UpsertAdminLLMUpstreamModelRequest = { platformModelName, upstreamModelName: row.upstreamModelName.trim(), - protocol: row.protocol || row.suggestedProtocol || undefined, kindsJSON: displayToKindsJson(row.kindsDisplay), status: row.routeStatus || "active", priority: row.priority || 1, weight: row.weight || 1, }; - operations.push(upsertAdminLLMUpstreamModel(token, upstream.id, payload)); - savedCount += 1; + const desiredProtocols = selectedProtocolsForSave(row); + if (desiredProtocols.length === 0) { + const keepRouteID = row.routeID || existingRouteIDs[0]; + upsertOperations.push(() => + upsertAdminLLMUpstreamModel(token, upstream.id, { + ...basePayload, + routeID: keepRouteID || undefined, + }), + ); + savedCount += 1; + for (const routeID of existingRouteIDs) { + if (routeID === keepRouteID) continue; + deleteOperations.push(() => deleteAdminLLMUpstreamModel(token, upstream.id, routeID)); + deletedCount += 1; + } + continue; + } + + const desiredSet = new Set(desiredProtocols); + const reusableRouteIDs = Object.entries(row.routeIDsByProtocol) + .filter(([protocol]) => !desiredSet.has(protocol as AdminLLMAdapter)) + .map(([, routeID]) => routeID) + .filter((routeID) => routeID > 0); + const reusedRouteIDs = new Set(); + for (const protocol of desiredProtocols) { + const existingRouteID = row.routeIDsByProtocol[protocol]; + const routeID = existingRouteID || reusableRouteIDs.shift(); + if (routeID) { + reusedRouteIDs.add(routeID); + } + upsertOperations.push(() => + upsertAdminLLMUpstreamModel(token, upstream.id, { + ...basePayload, + routeID, + protocol, + }), + ); + savedCount += 1; + } + for (const [protocol, routeID] of Object.entries(row.routeIDsByProtocol)) { + if (desiredSet.has(protocol as AdminLLMAdapter) || reusedRouteIDs.has(routeID)) continue; + deleteOperations.push(() => deleteAdminLLMUpstreamModel(token, upstream.id, routeID)); + deletedCount += 1; + } } - if (operations.length === 0) { + if (deleteOperations.length === 0 && upsertOperations.length === 0) { toast.info(t("modelsDialog.noSavableChanges")); await loadBindings(); return; } - await Promise.all(operations); + await runOperationsInOrder(upsertOperations); + await runOperationsInOrder(deleteOperations); if (savedCount > 0 && deletedCount > 0) { toast.success(t("modelsDialog.savedAndDeleted", { savedCount, deletedCount })); } else if (deletedCount > 0) { @@ -1055,7 +1191,7 @@ export function UpstreamModelsDialog({ await loadBindings(); onUpstreamUpdated({ ...upstream }); } catch (err) { - toast.error(resolveErrorMessage(err, t("toast.updateFailed"))); + toast.error(t("toast.updateFailed"), { description: resolveErrorMessage(err) }); } finally { setSaving(false); } @@ -1063,6 +1199,13 @@ export function UpstreamModelsDialog({ const dirtyCount = rows.filter((r) => r.isDirty).length; const selectedCount = selected.size; + const selectedRouteCount = React.useMemo( + () => + rows + .filter((row) => selected.has(row.draftKey)) + .reduce((count, row) => count + routeIDsForRow(row).length, 0), + [rows, selected], + ); return ( <> @@ -1161,27 +1304,23 @@ export function UpstreamModelsDialog({ } label={t("actions.apply")} - onApply={() => setBulkPatchConfirm({ patch: { protocol: bulkProtocol } })} + onApply={() => + setBulkPatchConfirm({ + patch: { + protocols: bulkProtocols, + protocol: bulkProtocols[0] ?? "", + kindsDisplay: resolveKindsDisplayForProtocols(bulkProtocols, bulkKindsDisplay), + }, + }) + } disabled={selectedCount === 0} > - + className="h-7 w-full px-2 text-[11px]" + /> {t("modelsDialog.routeStatus")} {t("modelsDialog.upstreamModelName")} {t("modelsDialog.platformModel")} - {t("modelsDialog.protocol")} + {t("modelsDialog.protocol")} {t("modelsDialog.kind")} @@ -1249,7 +1388,7 @@ export function UpstreamModelsDialog({ 0 && selected.has(row.routeID)} + isSelected={selected.has(row.draftKey)} onSelect={handleSelectOne} onUpdate={updateRow} /> @@ -1312,7 +1451,7 @@ export function UpstreamModelsDialog({ {t("modelsDialog.batchDeleteTitle")} - {t("modelsDialog.batchDeleteDescription", { count: selected.size })} + {t("modelsDialog.batchDeleteDescription", { count: selectedCount })} @@ -1323,14 +1462,14 @@ export function UpstreamModelsDialog({ {commonT("actions.cancel")} { event.preventDefault(); void handleDeleteSelected(); }} className="bg-destructive text-destructive-foreground hover:bg-destructive/90" > - {deleting ? {t("modelsDialog.deleting")} : t("modelsDialog.confirmDelete", { count: selected.size })} + {deleting ? {t("modelsDialog.deleting")} : t("modelsDialog.confirmDelete", { count: selectedCount })} diff --git a/frontend/features/admin/components/sections/upstreams/upstream-sheet.tsx b/frontend/features/admin/components/sections/upstreams/upstream-sheet.tsx index 84779923..82fb8dfb 100644 --- a/frontend/features/admin/components/sections/upstreams/upstream-sheet.tsx +++ b/frontend/features/admin/components/sections/upstreams/upstream-sheet.tsx @@ -76,6 +76,8 @@ const PROTOCOL_OPTIONS_BY_KIND: Record<(typeof PROTOCOL_DEFAULT_KINDS)[number], ], image_edit: [ "openai_image_edits", + "google_image_generation", + "xai_image_edits", ], video_gen: [ "openai_video_generations", diff --git a/frontend/features/admin/hooks/use-admin-models.ts b/frontend/features/admin/hooks/use-admin-models.ts index d53a7564..2008352d 100644 --- a/frontend/features/admin/hooks/use-admin-models.ts +++ b/frontend/features/admin/hooks/use-admin-models.ts @@ -6,8 +6,8 @@ import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; import { listAdminLLMModelUpstreamSources, listAdminLLMModels, + upsertAdminLLMUpstreamModel, updateAdminLLMModel, - updateAdminLLMModelUpstreamSource, } from "@/features/admin/api"; import type { AdminLLMAdapter, @@ -22,6 +22,7 @@ import { resolveErrorMessage, type ModelSortValue, } from "@/features/admin/types/llm"; +import { resolveKindsDisplayForProtocols } from "@/features/admin/utils/llm-display"; import { patchByID, removeByID, removeManyByID, replaceByID } from "@/shared/lib/optimistic-list"; type UseAdminModelsState = { @@ -458,9 +459,10 @@ export function useAdminModels(): UseAdminModelsState { const rollbackModels = targets.map((item) => items.find((current) => current.id === item.id) ?? item); const targetIDs = new Set(targets.map((item) => item.id)); const nextProtocolsJSON = JSON.stringify([nextProtocol]); + const nextKindsJSON = displayToKindsJson(resolveKindsDisplayForProtocols([nextProtocol])); setBatchApplying(true); setItems((current) => - current.map((item) => (targetIDs.has(item.id) ? { ...item, protocolsJSON: nextProtocolsJSON } : item)), + current.map((item) => (targetIDs.has(item.id) ? { ...item, protocolsJSON: nextProtocolsJSON, kindsJSON: nextKindsJSON } : item)), ); try { const results = await Promise.allSettled( @@ -469,20 +471,29 @@ export function useAdminModels(): UseAdminModelsState { if (sources.results.length === 0) { throw new Error("model upstream sources not found"); } - const sourceResults = await Promise.allSettled( - sources.results.map((source) => - updateAdminLLMModelUpstreamSource(token, model.id, source.id, { protocol: nextProtocol }), - ), - ); - const hasFailedSource = sourceResults.some((result) => result.status === "rejected"); - if (hasFailedSource) { - throw new Error("failed to update model source protocol"); + for (const source of sources.results) { + await upsertAdminLLMUpstreamModel(token, source.upstreamID, { + routeID: source.id, + platformModelName: model.platformModelName, + upstreamModelName: source.upstreamModelName, + protocol: nextProtocol, + kindsJSON: nextKindsJSON, + status: source.status, + priority: source.priority, + weight: source.weight, + }); } - return model; + return { ...model, kindsJSON: nextKindsJSON, protocolsJSON: nextProtocolsJSON }; }), ); const failedModels = targets.filter((_, index) => results[index]?.status === "rejected"); const successModels = targets.filter((_, index) => results[index]?.status === "fulfilled"); + const successResponses = results + .filter((result): result is PromiseFulfilledResult => result.status === "fulfilled") + .map((result) => result.value); + setItems((current) => + successResponses.reduce((next, model) => replaceByID(next, model.id, (item) => item.id, model), current), + ); if (failedModels.length > 0) { const failedIDs = new Set(failedModels.map((item) => item.id)); setItems((current) => diff --git a/frontend/features/admin/model/conversation-settings.ts b/frontend/features/admin/model/conversation-settings.ts index 7fb27642..65ba869a 100644 --- a/frontend/features/admin/model/conversation-settings.ts +++ b/frontend/features/admin/model/conversation-settings.ts @@ -49,6 +49,31 @@ export const DEFAULT_MODEL_OPTION_ALLOWED_PATHS = `{ "reasoning.summary", "text.verbosity" ], + "openai_image_generations": [ + "background", + "moderation", + "n", + "output_compression", + "output_format", + "partial_images", + "quality", + "response_format", + "size", + "style", + "user" + ], + "openai_image_edits": [ + "background", + "input_fidelity", + "n", + "output_compression", + "output_format", + "partial_images", + "quality", + "response_format", + "size", + "user" + ], "google_image_generation": [ "aspect_ratio", "aspectRatio", @@ -78,6 +103,12 @@ export const DEFAULT_MODEL_OPTION_ALLOWED_PATHS = `{ "resolution", "response_format" ], + "xai_image_edits": [ + "aspect_ratio", + "n", + "resolution", + "response_format" + ], "gemini_generate_content": [ "generationConfig.temperature", "generationConfig.topP", diff --git a/frontend/features/admin/model/upstream-models.ts b/frontend/features/admin/model/upstream-models.ts index db02270d..3a4e79a3 100644 --- a/frontend/features/admin/model/upstream-models.ts +++ b/frontend/features/admin/model/upstream-models.ts @@ -12,12 +12,14 @@ export type RowDraft = AdminLLMUpstreamModelDTO & { isDirty: boolean; kindsDisplay: string; platformModelNameDraft: string; + protocols: AdminLLMAdapter[]; + routeIDsByProtocol: Record; }; export type NewBindingFormState = { upstreamModelName: string; platformModelName: string; - protocol: AdminLLMAdapter; + protocols: AdminLLMAdapter[]; kindsDisplay: string; status: AdminLLMStatus; }; @@ -25,7 +27,7 @@ export type NewBindingFormState = { export const DEFAULT_NEW_BINDING: NewBindingFormState = { upstreamModelName: "", platformModelName: "", - protocol: "openai_responses", + protocols: ["openai_responses"], kindsDisplay: "chat", status: "active", }; @@ -45,13 +47,39 @@ export function displayToKindsJson(display: string): string { } export function buildRowDrafts(items: AdminLLMUpstreamModelDTO[]): RowDraft[] { - return items.map((item) => ({ - ...item, - draftKey: item.routeID > 0 ? `route:${item.routeID}` : `upstream:${item.upstreamID}:${item.upstreamModelName}`, - platformModelNameDraft: item.platformModelName, - isDirty: false, - kindsDisplay: kindsJsonToDisplay(item.upstreamModelKindsJSON || item.modelKindsJSON), - })); + const grouped = new Map(); + for (const item of items) { + const platformModelName = item.platformModelName || ""; + const draftKey = platformModelName + ? `binding:${item.upstreamID}:${item.upstreamModelName}:${platformModelName}` + : `upstream:${item.upstreamID}:${item.upstreamModelName}`; + const existing = grouped.get(draftKey); + if (existing) { + if (item.protocol && !existing.protocols.includes(item.protocol)) { + existing.protocols.push(item.protocol); + } + if (item.protocol && item.routeID > 0) { + existing.routeIDsByProtocol[item.protocol] = item.routeID; + } + if (item.routeStatus === "active") { + existing.routeStatus = "active"; + } + if (item.routeID > 0 && (!existing.routeID || item.routeID < existing.routeID)) { + existing.routeID = item.routeID; + } + continue; + } + grouped.set(draftKey, { + ...item, + draftKey, + platformModelNameDraft: platformModelName, + isDirty: false, + kindsDisplay: kindsJsonToDisplay(item.upstreamModelKindsJSON || item.modelKindsJSON), + protocols: item.protocol ? [item.protocol] : [], + routeIDsByProtocol: item.protocol && item.routeID > 0 ? { [item.protocol]: item.routeID } : {}, + }); + } + return Array.from(grouped.values()); } export type UpstreamModelMessages = { @@ -85,12 +113,15 @@ export function validateRowDrafts( if (!platformModelName) { continue; } - const bindingKey = `${upstreamModelName}\u0000${platformModelName}`; - const existingOwner = bindingOwners.get(bindingKey); - if (existingOwner && existingOwner !== row.draftKey) { - return messages.duplicateBinding(upstreamModelName, platformModelName); + const protocols = row.protocols.length > 0 ? row.protocols : [row.protocol || row.suggestedProtocol || ""]; + for (const protocol of protocols) { + const bindingKey = `${upstreamModelName}\u0000${platformModelName}\u0000${protocol}`; + const existingOwner = bindingOwners.get(bindingKey); + if (existingOwner && existingOwner !== row.draftKey) { + return messages.duplicateBinding(upstreamModelName, platformModelName); + } + bindingOwners.set(bindingKey, row.draftKey); } - bindingOwners.set(bindingKey, row.draftKey); } return undefined; } diff --git a/frontend/features/admin/types/llm.ts b/frontend/features/admin/types/llm.ts index 9aad18fc..9b5a8661 100644 --- a/frontend/features/admin/types/llm.ts +++ b/frontend/features/admin/types/llm.ts @@ -35,6 +35,7 @@ export const ADAPTER_LABELS: Record = { google_image_generation: resolveProtocolLabel("google_image_generation"), xai_responses: resolveProtocolLabel("xai_responses"), xai_image: resolveProtocolLabel("xai_image"), + xai_image_edits: resolveProtocolLabel("xai_image_edits"), }; // --------------------------------------------------------------------------- diff --git a/frontend/features/admin/utils/llm-display.ts b/frontend/features/admin/utils/llm-display.ts index c0d8e726..03a765c1 100644 --- a/frontend/features/admin/utils/llm-display.ts +++ b/frontend/features/admin/utils/llm-display.ts @@ -6,6 +6,7 @@ import { Video, } from "lucide-react"; import type { LucideIcon } from "lucide-react"; +import type { AdminLLMAdapter } from "@/features/admin/api/llm.types"; export const MODEL_KIND_META: Record< string, @@ -27,23 +28,39 @@ export const COMPATIBLE_OPTIONS = [ { label: "Custom", value: "custom" }, ] as const; -export const PROTOCOL_OPTIONS = [ - { value: "openai_responses", label: "Responses API (OpenAI)" }, - { value: "openai_chat_completions", label: "Chat Completions (OpenAI)" }, - { value: "openai_image_generations", label: "Image Generations (OpenAI)" }, - { value: "anthropic_messages", label: "Messages (Anthropic)" }, - { value: "google_generate_content", label: "Generate Content (Google)" }, - { value: "google_image_generation", label: "Image Generation (Google)" }, - { value: "xai_responses", label: "Responses (xAI)" }, - { value: "xai_image", label: "Image Generation (xAI)" }, +type ProtocolOption = { + value: AdminLLMAdapter; + label: string; + kinds: readonly string[]; +}; + +export const PROTOCOL_OPTIONS: ReadonlyArray = [ + { value: "openai_responses", label: "Responses API (OpenAI)", kinds: ["chat"] }, + { value: "openai_chat_completions", label: "Chat Completions (OpenAI)", kinds: ["chat"] }, + { value: "openai_image_generations", label: "Images Generations (OpenAI)", kinds: ["image_gen"] }, + { value: "openai_image_edits", label: "Images Edits (OpenAI)", kinds: ["image_edit"] }, + { value: "openai_video_generations", label: "Video Generations (OpenAI)", kinds: ["video_gen"] }, + { value: "anthropic_messages", label: "Messages (Anthropic)", kinds: ["chat"] }, + { value: "google_generate_content", label: "Generate Content (Google)", kinds: ["chat"] }, + { value: "google_image_generation", label: "Image Generation (Google)", kinds: ["image_gen", "image_edit"] }, + { value: "xai_responses", label: "Responses (xAI)", kinds: ["chat"] }, + { value: "xai_image", label: "Images Generations (xAI)", kinds: ["image_gen"] }, + { value: "xai_image_edits", label: "Images Edits (xAI)", kinds: ["image_edit"] }, ] as const; const PROTOCOL_LABELS: Record = { ...Object.fromEntries(PROTOCOL_OPTIONS.map((item) => [item.value, item.label])), - openai_image_edits: "Image Edits (OpenAI)", - openai_video_generations: "Video Generations (OpenAI)", }; +const PROTOCOL_KINDS: Record = { + ...Object.fromEntries(PROTOCOL_OPTIONS.map((item) => [item.value, item.kinds])), +}; + +const IMAGE_ROUTE_PROTOCOL_PAIRS: ReadonlyArray = [ + ["openai_image_generations", "openai_image_edits"], + ["xai_image", "xai_image_edits"], +]; + const LLM_STATUS_LABELS: Record = { active: "Enabled", inactive: "Disabled", @@ -65,6 +82,39 @@ export function resolveProtocolLabel(protocol: string): string { return PROTOCOL_LABELS[protocol] ?? protocol; } +export function isSupportedRouteProtocolSelection(protocols: readonly AdminLLMAdapter[]): boolean { + const uniqueProtocols = Array.from(new Set(protocols)); + if (uniqueProtocols.length <= 1) { + return true; + } + return uniqueProtocols.length === 2 && IMAGE_ROUTE_PROTOCOL_PAIRS.some(([generationProtocol, editProtocol]) => + uniqueProtocols.includes(generationProtocol) && uniqueProtocols.includes(editProtocol), + ); +} + +export function resolveNextRouteProtocolSelection( + currentProtocols: readonly AdminLLMAdapter[], + protocol: AdminLLMAdapter, +): AdminLLMAdapter[] { + const current = Array.from(new Set(currentProtocols)); + if (current.includes(protocol)) { + return current.filter((item) => item !== protocol); + } + const candidate = [...current, protocol]; + if (isSupportedRouteProtocolSelection(candidate)) { + return candidate; + } + return [protocol]; +} + +export function resolveKindsDisplayForProtocols( + protocols: readonly AdminLLMAdapter[], + fallbackDisplay = "chat", +): string { + const kinds = Array.from(new Set(protocols.flatMap((protocol) => PROTOCOL_KINDS[protocol] ?? []))); + return kinds.length > 0 ? kinds.join(",") : fallbackDisplay; +} + export function resolveCompatibleLabel(compatible: string): string { return COMPATIBLE_OPTIONS.find((item) => item.value === compatible)?.label ?? (compatible || "-"); } diff --git a/frontend/features/chat/components/app-chat-area.tsx b/frontend/features/chat/components/app-chat-area.tsx index da71e63a..18a8d5ec 100644 --- a/frontend/features/chat/components/app-chat-area.tsx +++ b/frontend/features/chat/components/app-chat-area.tsx @@ -3,13 +3,14 @@ import * as React from "react"; import { useRouter, useSearchParams } from "next/navigation"; import { useTranslations } from "next-intl"; +import { toast } from "sonner"; import { ChatArea, ChatAreaLoadError, ChatAreaSkeleton } from "@/features/chat/components/sections/chat-area"; import { ChatEmptyState } from "@/features/chat/components/sections/chat-empty"; import { useChatSession } from "@/features/chat/context/chat-session-context"; import { useChatAttachments } from "@/features/chat/hooks/use-chat-attachments"; import { useConversationComposerState } from "@/features/chat/hooks/use-conversation-composer-state"; -import type { ChatAreaMessage } from "@/features/chat/types/messages"; +import type { ChatAreaMessage, MessageAttachment } from "@/features/chat/types/messages"; import { useChatModelOptions } from "@/features/chat/hooks/use-chat-model-options"; import { useChatRuntime } from "@/features/chat/hooks/use-chat-runtime"; import { useChatScrollController } from "@/features/chat/hooks/use-chat-scroll-controller"; @@ -26,6 +27,7 @@ import { } from "@/features/chat/model/conversation-options"; import { useSidebarRecents } from "@/features/recent/context/sidebar-recents-context"; import { useChatData } from "@/features/chat/hooks/use-chat-data"; +import { toPendingAttachment } from "@/features/chat/model/message-submit"; import { listAvailableMCPTools } from "@/shared/api/mcp"; import { resolveAccessToken } from "@/shared/auth/resolve-access-token"; import type { ConversationOptions } from "@/shared/api/conversation.types"; @@ -350,6 +352,50 @@ export function AppChatArea() { streamingTraceText, }); + const onEditGeneratedImageAttachment = React.useCallback( + (attachment: MessageAttachment, sourceModelName?: string) => { + const alreadyAttached = attachments.some((item) => item.fileID === attachment.fileID); + if (!alreadyAttached && maxFilesPerMessage > 0 && attachments.length >= maxFilesPerMessage) { + toast.error(t("attachments.limitReached"), { + description: t("attachments.maxUploadFiles", { count: maxFilesPerMessage }), + }); + return; + } + + const pendingAttachment = toPendingAttachment(attachment); + setAttachments((previous) => { + if (previous.some((item) => item.fileID === pendingAttachment.fileID)) { + return previous; + } + return [...previous, pendingAttachment]; + }); + + const selectedSupportsImageEdit = selectedModel?.kinds.includes("image_edit") ?? false; + if (!selectedSupportsImageEdit) { + const normalizedSourceModelName = sourceModelName?.trim() || ""; + const sourceModel = modelOptions.find( + (item) => item.platformModelName === normalizedSourceModelName && item.kinds.includes("image_edit"), + ); + const fallbackModel = sourceModel ?? modelOptions.find((item) => item.kinds.includes("image_edit")); + if (fallbackModel) { + setSelectedPlatformModelName(fallbackModel.platformModelName); + } + } + + window.requestAnimationFrame(onScrollToLatest); + }, + [ + attachments, + maxFilesPerMessage, + modelOptions, + onScrollToLatest, + selectedModel, + setAttachments, + setSelectedPlatformModelName, + t, + ], + ); + React.useEffect(() => { setManualConversationTitle(""); }, [conversationID]); @@ -530,6 +576,7 @@ export function AppChatArea() { onRetryUserMessage={onRetryUserMessage} onRetryAssistantMessage={onRetryAssistantMessage} onEditUserMessage={onEditUserMessage} + onEditImageAttachment={onEditGeneratedImageAttachment} onCycleMessageBranch={onCycleMessageBranch} onToggleStar={onToggleActiveConversationStar} onRename={onRenameActiveConversation} diff --git a/frontend/features/chat/components/markdown/streamdown-components.tsx b/frontend/features/chat/components/markdown/streamdown-components.tsx index 0bda9c2c..590e6522 100644 --- a/frontend/features/chat/components/markdown/streamdown-components.tsx +++ b/frontend/features/chat/components/markdown/streamdown-components.tsx @@ -1,13 +1,14 @@ "use client"; import * as React from "react"; -import { Check, Copy, CornerUpLeft, Download, Maximize2 } from "lucide-react"; +import { Check, Copy, CornerUpLeft, Download, Maximize2, WandSparkles } from "lucide-react"; import { useTranslations } from "next-intl"; import { ChevronDown } from "@/components/animate-ui/icons/chevron-down"; import { ChevronUp } from "@/components/animate-ui/icons/chevron-up"; import { Button } from "@/components/ui/button"; import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { downloadMarkdownImageSource, loadProtectedMarkdownImageBlobURL, @@ -49,6 +50,11 @@ type MarkdownImageProps = React.ImgHTMLAttributes & { src?: string; }; +export type MarkdownImageActions = { + canEditImage?: (src: string) => boolean; + onEditImage?: (src: string) => void; +}; + type MarkdownParagraphProps = React.HTMLAttributes & { children?: React.ReactNode; }; @@ -59,6 +65,7 @@ type MarkdownHeadingProps = React.HTMLAttributes & { const StreamdownLinkContext = React.createContext(false); const FootnoteBackrefGroupContext = React.createContext(false); +export const MarkdownImageActionsContext = React.createContext(null); function resolveLinkKind(href: string): ResolvedLinkKind { if (href.startsWith("#")) { @@ -488,6 +495,7 @@ export function MarkdownLink({ children, className, href, onClick, ...props }: M export function MarkdownImage({ alt, className, onError, onLoad, src, ...props }: MarkdownImageProps) { const t = useTranslations("chat.markdown"); const insideLink = React.useContext(StreamdownLinkContext); + const imageActions = React.useContext(MarkdownImageActionsContext); const [loaded, setLoaded] = React.useState(false); const [failed, setFailed] = React.useState(false); const [previewOpen, setPreviewOpen] = React.useState(false); @@ -569,6 +577,7 @@ export function MarkdownImage({ alt, className, onError, onLoad, src, ...props } }, [alt, resolvedSrc, src]); const canUseImageActions = !insideLink && !failed && Boolean(displaySrc); + const canEditImage = Boolean(src && imageActions?.onEditImage && (imageActions.canEditImage?.(src) ?? true)); if (!src) { return null; @@ -604,22 +613,47 @@ export function MarkdownImage({ alt, className, onError, onLoad, src, ...props } loaded ? "opacity-100" : "opacity-0", )} > - - + + + + + {t("previewImage")} + + {canEditImage ? ( + + + + + {t("editImage")} + + ) : null} + + + + + {t("downloadImage")} + ) : null} diff --git a/frontend/features/chat/components/markdown/streamdown-render.tsx b/frontend/features/chat/components/markdown/streamdown-render.tsx index 7be2b94d..5bfb8e8c 100644 --- a/frontend/features/chat/components/markdown/streamdown-render.tsx +++ b/frontend/features/chat/components/markdown/streamdown-render.tsx @@ -16,10 +16,12 @@ import { cn } from "@/lib/utils"; import { CollapsibleCodePre, + MarkdownImageActionsContext, MarkdownImage, MarkdownLink, MarkdownParagraph, ThinkingHeading, + type MarkdownImageActions, } from "./streamdown-components"; import { normalizeContent, @@ -35,6 +37,7 @@ type StreamdownRenderProps = { className?: string; streaming?: boolean; variant?: "default" | "thinking"; + imageActions?: MarkdownImageActions; }; type StreamdownFeatureFlags = { @@ -370,6 +373,7 @@ export const StreamdownRender = React.memo(function StreamdownRender({ className, streaming = false, variant = "default", + imageActions, }: StreamdownRenderProps) { const normalizedContent = React.useMemo(() => normalizeStreamdownContent(content), [content]); const plugins = useStreamdownPlugins(normalizedContent); @@ -412,23 +416,24 @@ export const StreamdownRender = React.memo(function StreamdownRender({ /> ) : null} {markdownSegments.map((segment, index) => ( - - {segment.content} - + + + {segment.content} + + ))}
); diff --git a/frontend/features/chat/components/message/message-attachment.tsx b/frontend/features/chat/components/message/message-attachment.tsx index cfd3f06d..b0fe1617 100644 --- a/frontend/features/chat/components/message/message-attachment.tsx +++ b/frontend/features/chat/components/message/message-attachment.tsx @@ -38,28 +38,32 @@ function AttachmentCard({ const fileIcon = resolveFileIcon(att); return ( -
- +
+

+ {att.fileName} +

+
+ + {meta} + + + {ext} + +
+
+ + ); } diff --git a/frontend/features/chat/components/message/message-bot.tsx b/frontend/features/chat/components/message/message-bot.tsx index 88666b42..8a248fef 100644 --- a/frontend/features/chat/components/message/message-bot.tsx +++ b/frontend/features/chat/components/message/message-bot.tsx @@ -12,6 +12,7 @@ import type { AssistantReaction } from "@/features/chat/components/message/messa import type { ChatAreaMessage, ChatInlineAlert, + MessageAttachment, } from "@/features/chat/types/messages"; import { StreamdownRender } from "@/features/chat/components/markdown/streamdown-render"; import { @@ -32,6 +33,51 @@ import type { PreviewDialogFile } from "@/features/files/components/preview/file const EMPTY_TRACE_EVENTS: NonNullable["events"] = []; +function isEditableImageAttachment(attachment: MessageAttachment): boolean { + const mimeType = attachment.mimeType.toLowerCase(); + const detectedMime = attachment.detectedMime?.toLowerCase() || ""; + return ( + attachment.kind === "image" || + attachment.fileCategory === "image" || + mimeType.startsWith("image/") || + detectedMime.startsWith("image/") + ); +} + +function resolveFileIDFromImageSrc(src: string): string | null { + if (typeof window === "undefined") { + return null; + } + try { + const url = new URL(src, window.location.origin); + const match = url.pathname.match(/\/api\/v1\/files\/([^/]+)\/content$/); + return match?.[1] ? decodeURIComponent(match[1]) : null; + } catch { + return null; + } +} + +function resolveEditableImageAttachment( + src: string, + attachments: MessageAttachment[], + contentType: string | undefined, +): MessageAttachment | null { + if (attachments.length === 0) { + return null; + } + + const fileID = resolveFileIDFromImageSrc(src); + if (fileID) { + return attachments.find((attachment) => attachment.fileID === fileID) ?? null; + } + + if (contentType === "image" && attachments.length === 1) { + return attachments[0]; + } + + return null; +} + type ChatMessageBotProps = { item: ChatAreaMessage; busy: boolean; @@ -47,6 +93,7 @@ type ChatMessageBotProps = { showBillingCost?: boolean; readOnly?: boolean; attachmentContentLoader?: (file: PreviewDialogFile) => Promise; + onEditImageAttachment?: (attachment: MessageAttachment, sourceModelName?: string) => void; showBranchNavigator?: boolean; }; @@ -65,6 +112,7 @@ export function ChatMessageBot({ showBillingCost = false, readOnly = false, attachmentContentLoader, + onEditImageAttachment, showBranchNavigator = true, }: ChatMessageBotProps) { const onRetry = React.useCallback(() => { @@ -90,6 +138,34 @@ export function ChatMessageBot({ ); const hasTraceEvents = postProcessEvents.length > 0; const isImageGenerationLoading = item.contentType === "image" && item.isStreaming && !hasStreamdownContent; + const editableImageAttachments = React.useMemo( + () => (item.attachments ?? []).filter(isEditableImageAttachment), + [item.attachments], + ); + const getEditableImageAttachment = React.useCallback( + (src: string) => resolveEditableImageAttachment(src, editableImageAttachments, item.contentType), + [editableImageAttachments, item.contentType], + ); + const markdownImageActions = React.useMemo(() => { + if (readOnly || !onEditImageAttachment || editableImageAttachments.length === 0) { + return undefined; + } + return { + canEditImage: (src: string) => Boolean(getEditableImageAttachment(src)), + onEditImage: (src: string) => { + const attachment = getEditableImageAttachment(src); + if (attachment) { + onEditImageAttachment(attachment, item.platformModelName); + } + }, + }; + }, [ + editableImageAttachments.length, + getEditableImageAttachment, + item.platformModelName, + onEditImageAttachment, + readOnly, + ]); const activeThinkBlock = hasTraceEvents && upstreamThink?.status === "streaming" ? upstreamThink : undefined; const activeToolBlock = hasTraceEvents && toolTrace?.status === "streaming" ? toolTrace : undefined; const processAutoCollapseReady = Boolean(hasTraceEvents || upstreamThink || toolTrace || hasStreamdownContent || item.inlineAlert); @@ -135,7 +211,7 @@ export function ChatMessageBot({ ) : item.isStreaming && !hasStreamdownContent && !item.inlineAlert ? ( ) : hasStreamdownContent && markdownRender ? ( - + ) : hasStreamdownContent ? (

{item.content}

) : null} diff --git a/frontend/features/chat/components/sections/chat-area.tsx b/frontend/features/chat/components/sections/chat-area.tsx index 2af7d46e..90aede10 100644 --- a/frontend/features/chat/components/sections/chat-area.tsx +++ b/frontend/features/chat/components/sections/chat-area.tsx @@ -15,7 +15,7 @@ import { } from "@/features/chat/components/message/message-bot"; import { areChatAreaMessagesRenderEqual } from "@/features/chat/model/chat-message-render"; import { type AssistantReaction } from "@/features/chat/components/message/message-meta"; -import type { ChatAreaMessage } from "@/features/chat/types/messages"; +import type { ChatAreaMessage, MessageAttachment } from "@/features/chat/types/messages"; import { ChatMessageUser } from "@/features/chat/components/message/message-user"; import { StreamdownRender } from "@/features/chat/components/markdown/streamdown-render"; import { CenteredEmptyState } from "@/components/ui/empty-state"; @@ -73,6 +73,7 @@ type ChatAreaProps = { onRetryUserMessage: (message: ChatAreaMessage) => Promise | void; onRetryAssistantMessage: (message: ChatAreaMessage) => Promise | void; onEditUserMessage: (message: ChatAreaMessage, content: string) => Promise | boolean; + onEditImageAttachment?: (attachment: MessageAttachment, sourceModelName?: string) => void; onCycleMessageBranch: (parentPublicID: string | null, direction: "previous" | "next") => void; onToggleStar?: () => void | Promise; onRename?: (title: string) => void | Promise; @@ -103,6 +104,7 @@ const ChatMessageRow = React.memo(function ChatMessageRow({ onRetryUserMessage, onRetryAssistantMessage, onEditUserMessage, + onEditImageAttachment, onCycleMessageBranch, onReactAssistantMessage, markdownRender, @@ -117,6 +119,7 @@ const ChatMessageRow = React.memo(function ChatMessageRow({ onRetryUserMessage: (message: ChatAreaMessage) => Promise | void; onRetryAssistantMessage: (message: ChatAreaMessage) => Promise | void; onEditUserMessage: (message: ChatAreaMessage, content: string) => Promise | boolean; + onEditImageAttachment?: (attachment: MessageAttachment, sourceModelName?: string) => void; onCycleMessageBranch: (parentPublicID: string | null, direction: "previous" | "next") => void; onReactAssistantMessage: (publicID: string, reaction: AssistantReaction) => void; markdownRender: boolean; @@ -162,6 +165,7 @@ const ChatMessageRow = React.memo(function ChatMessageRow({ onCycleMessageBranch={onCycleMessageBranch} onReactAssistantMessage={onReactAssistantMessage} onCopy={() => void onCopy()} + onEditImageAttachment={onEditImageAttachment} markdownRender={markdownRender} showModelInfo={showModelInfo} showLatency={showLatency} @@ -191,6 +195,7 @@ const ChatMessageRow = React.memo(function ChatMessageRow({ previous.showLatency === next.showLatency && previous.showTokenUsage === next.showTokenUsage && previous.showBillingCost === next.showBillingCost && + previous.onEditImageAttachment === next.onEditImageAttachment && areChatAreaMessagesRenderEqual(previous.item, next.item) )); @@ -208,6 +213,7 @@ export function ChatArea({ onRetryUserMessage, onRetryAssistantMessage, onEditUserMessage, + onEditImageAttachment, onCycleMessageBranch, onToggleStar, onRename, @@ -226,8 +232,12 @@ export function ChatArea({ const stableOnRetryUserMessage = useStableEvent(onRetryUserMessage); const stableOnRetryAssistantMessage = useStableEvent(onRetryAssistantMessage); const stableOnEditUserMessage = useStableEvent(onEditUserMessage); + const stableOnEditImageAttachment = useStableEvent((attachment: MessageAttachment, sourceModelName?: string) => { + onEditImageAttachment?.(attachment, sourceModelName); + }); const stableOnCycleMessageBranch = useStableEvent(onCycleMessageBranch); const stableOnReactAssistantMessage = useStableEvent(onReactAssistantMessage); + const editImageAttachmentHandler = onEditImageAttachment ? stableOnEditImageAttachment : undefined; const shareLabel = shareActive ? t("manageShare") : t("shareConversation"); return ( @@ -293,6 +303,7 @@ export function ChatArea({ onRetryUserMessage={stableOnRetryUserMessage} onRetryAssistantMessage={stableOnRetryAssistantMessage} onEditUserMessage={stableOnEditUserMessage} + onEditImageAttachment={editImageAttachmentHandler} onCycleMessageBranch={stableOnCycleMessageBranch} onReactAssistantMessage={stableOnReactAssistantMessage} markdownRender={markdownRender} diff --git a/frontend/features/chat/components/sections/chat-input.tsx b/frontend/features/chat/components/sections/chat-input.tsx index 5480b62f..815dbe84 100644 --- a/frontend/features/chat/components/sections/chat-input.tsx +++ b/frontend/features/chat/components/sections/chat-input.tsx @@ -2,7 +2,7 @@ import * as React from "react"; import dynamic from "next/dynamic"; -import { Check, CircleHelp, Info } from "lucide-react"; +import { Check, CircleHelp, Image, ImageOff, ImagePlus, Info } from "lucide-react"; import { useTranslations } from "next-intl"; import { toast } from "sonner"; @@ -27,7 +27,8 @@ import { isReservedConversationOptionKey, sanitizeConversationOptions, } from "@/features/chat/model/conversation-options"; -import { isMediaSubmitTask, resolveChatSubmitTask } from "@/features/chat/model/chat-task"; +import type { ChatSubmitDecision } from "@/features/chat/model/chat-task"; +import { isMediaSubmitTask, resolveChatSubmitDecision } from "@/features/chat/model/chat-task"; import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; import { @@ -131,6 +132,14 @@ type NativeToolOption = { payload?: Record; }; +type ComposerModeIndicator = { + label: string; + intro: string; + description: string; + icon: React.ComponentType<{ className?: string; strokeWidth?: number }>; + tone: "default" | "warning"; +}; + type ModelOptionFilterStatus = "passed" | "filtered" | "unknown"; const OPTION_LABEL_KEYS = new Set([ @@ -495,16 +504,55 @@ const PROTOCOL_LABELS: Record = { google_generate_content: "Generate Content", google_image_generation: "Image Generation", openai_chat_completions: "Chat Completions", - openai_image_edits: "Image Edits", - openai_image_generations: "Image Generations", + openai_image_edits: "Images Edits", + openai_image_generations: "Images Generations", openai_responses: "Responses", openai_video_generations: "Video Generations", replicate_predictions: "Predictions", stability_ai_generate: "Image Generation", - xai_image: "Image Generation", + xai_image: "Images Generations", + xai_image_edits: "Images Edits", xai_responses: "xAI Responses", }; +function resolveComposerModeIndicator( + decision: ChatSubmitDecision, + t: (key: string) => string, +): ComposerModeIndicator | null { + if (decision.blockedReason === "image_task_rejects_non_image_attachments") { + return { + label: t("mediaMode.invalidFile"), + intro: t("mediaMode.invalidFileIntro"), + description: t(`mediaMode.blockedDescriptions.${decision.blockedReason}`), + icon: ImageOff, + tone: "warning", + }; + } + if (decision.task === "image_generation") { + return { + label: t("mediaMode.imageGeneration"), + intro: t("mediaMode.imageGenerationIntro"), + description: decision.blockedReason + ? t(`mediaMode.blockedDescriptions.${decision.blockedReason}`) + : t("mediaMode.imageGenerationDescription"), + icon: Image, + tone: "default", + }; + } + if (decision.task === "image_edit") { + return { + label: t("mediaMode.imageEdit"), + intro: t("mediaMode.imageEditIntro"), + description: decision.blockedReason + ? t(`mediaMode.blockedDescriptions.${decision.blockedReason}`) + : t("mediaMode.imageEditDescription"), + icon: ImagePlus, + tone: "default", + }; + } + return null; +} + function stringifyOptions(value: ConversationOptions): string { if (Object.keys(value).length === 0) { return ""; @@ -855,8 +903,11 @@ function ChatInputComponent({ ); const selectedProtocol = selectedModel?.protocols[0]?.trim() ?? ""; const selectedProtocolLabel = selectedProtocol ? resolveProtocolLabel(selectedProtocol) : ""; - const submitTask = resolveChatSubmitTask(selectedModel, attachments); + const submitDecision = resolveChatSubmitDecision(selectedModel, attachments); + const submitTask = submitDecision.task; const isMediaMode = isMediaSubmitTask(submitTask); + const composerModeIndicator = resolveComposerModeIndicator(submitDecision, tComposer); + const ComposerModeIcon = composerModeIndicator?.icon; const modelOptionPolicyDisabled = modelOptionPolicy?.mode?.trim() === "disabled"; const selectedToolIDSet = React.useMemo(() => new Set(selectedToolIDs), [selectedToolIDs]); const selectedToolCount = selectedToolIDs.length; @@ -1610,6 +1661,26 @@ function ChatInputComponent({
+ {composerModeIndicator && ComposerModeIcon ? ( + + + + + {composerModeIndicator.label} + + + + {composerModeIndicator.intro} {composerModeIndicator.description} + + + ) : null} string, +): string { + return t(`mediaInputBlocked.${reason}`); +} + function resolveImageLoadingAspectRatio(options: ConversationOptions): ImageLoadingAspectRatio { const rawSize = typeof options.size === "string" ? options.size.trim() : ""; const match = rawSize.match(/^(\d+)\s*x\s*(\d+)$/i); @@ -290,7 +298,14 @@ export function useChatMessageSubmit({ description: t("attachmentsTruncatedDescription", { count: maxFilesPerMessage }), }); } - const submitTask = resolveChatSubmitTask(selectedModel, effectiveAttachments); + const submitDecision = resolveChatSubmitDecision(selectedModel, effectiveAttachments); + if (submitDecision.blockedReason) { + toast.error(t("mediaInputUnsupported"), { + description: resolveSubmitBlockDescription(submitDecision.blockedReason, t), + }); + return false; + } + const submitTask = submitDecision.task; if (!requestPlatformModelName) { toast.error(t("noModel"), { description: t("selectModelFirst") }); return false; diff --git a/frontend/features/chat/model/chat-task.ts b/frontend/features/chat/model/chat-task.ts index 0869b420..b02243e4 100644 --- a/frontend/features/chat/model/chat-task.ts +++ b/frontend/features/chat/model/chat-task.ts @@ -1,20 +1,105 @@ import type { ChatModelOption, PendingAttachment } from "@/features/chat/types/chat-runtime"; export type ChatSubmitTask = "chat" | "image_generation" | "image_edit"; +export type ChatSubmitBlockReason = + | "image_edit_input_required" + | "image_edit_unsupported" + | "image_generation_rejects_attachments" + | "image_task_rejects_non_image_attachments" + | "model_task_unsupported"; -export function resolveChatSubmitTask( +export type ChatSubmitDecision = { + task: ChatSubmitTask; + blockedReason: ChatSubmitBlockReason | null; + attachmentCount: number; + imageAttachmentCount: number; + nonImageAttachmentCount: number; + supportsChat: boolean; + supportsImageGeneration: boolean; + supportsImageEdit: boolean; +}; + +function isImageAttachment(item: PendingAttachment): boolean { + return item.fileCategory === "image" || item.mimeType.toLowerCase().startsWith("image/"); +} + +function buildDecision( + task: ChatSubmitTask, + blockedReason: ChatSubmitBlockReason | null, + input: Omit, +): ChatSubmitDecision { + return { + task, + blockedReason, + ...input, + }; +} + +export function resolveChatSubmitDecision( model: ChatModelOption | null, attachments: PendingAttachment[], -): ChatSubmitTask { +): ChatSubmitDecision { const kinds = new Set(model?.kinds ?? []); - const hasImageAttachment = attachments.some((item) => item.fileCategory === "image"); - if (hasImageAttachment && kinds.has("image_edit")) { - return "image_edit"; + const attachmentCount = attachments.length; + const imageAttachmentCount = attachments.filter(isImageAttachment).length; + const nonImageAttachmentCount = attachmentCount - imageAttachmentCount; + const supportsChat = kinds.size === 0 || kinds.has("chat") || kinds.has("audio"); + const supportsImageGeneration = kinds.has("image_gen"); + const supportsImageEdit = kinds.has("image_edit"); + const baseDecision = { + attachmentCount, + imageAttachmentCount, + nonImageAttachmentCount, + supportsChat, + supportsImageGeneration, + supportsImageEdit, + }; + + if ( + nonImageAttachmentCount > 0 && + (supportsImageGeneration || supportsImageEdit) && + (imageAttachmentCount > 0 || !supportsChat) + ) { + if (imageAttachmentCount > 0 && supportsImageEdit) { + return buildDecision("image_edit", "image_task_rejects_non_image_attachments", baseDecision); + } + if (supportsImageGeneration) { + return buildDecision("image_generation", "image_task_rejects_non_image_attachments", baseDecision); + } + return buildDecision("chat", "image_task_rejects_non_image_attachments", baseDecision); + } + + if (imageAttachmentCount > 0) { + if (supportsImageEdit) { + return buildDecision("image_edit", null, baseDecision); + } + if (supportsChat) { + return buildDecision("chat", null, baseDecision); + } + return buildDecision("chat", "image_edit_unsupported", baseDecision); + } + + if (attachmentCount > 0) { + if (supportsChat) { + return buildDecision("chat", null, baseDecision); + } + if (supportsImageGeneration) { + return buildDecision("image_generation", "image_generation_rejects_attachments", baseDecision); + } + return buildDecision("chat", "model_task_unsupported", baseDecision); } - if (!hasImageAttachment && kinds.has("image_gen")) { - return "image_generation"; + + if (supportsImageGeneration) { + return buildDecision("image_generation", null, baseDecision); } - return "chat"; + if (supportsImageEdit && !supportsChat) { + return buildDecision("image_edit", "image_edit_input_required", baseDecision); + } + if (!supportsChat) { + return buildDecision("chat", "model_task_unsupported", baseDecision); + } + + return buildDecision("chat", null, baseDecision); } export function isMediaSubmitTask(task: ChatSubmitTask): boolean { diff --git a/frontend/features/chat/model/message-submit.ts b/frontend/features/chat/model/message-submit.ts index 37f3f204..6af7d2e1 100644 --- a/frontend/features/chat/model/message-submit.ts +++ b/frontend/features/chat/model/message-submit.ts @@ -1,4 +1,4 @@ -import type { ChatAreaMessage, ChatMessageProcessTrace } from "@/features/chat/types/messages"; +import type { ChatAreaMessage, ChatMessageProcessTrace, MessageAttachment } from "@/features/chat/types/messages"; import type { PendingAttachment } from "@/features/chat/types/chat-runtime"; import type { MessageProcessTraceDTO, @@ -9,7 +9,11 @@ export function toPendingAttachments(message: ChatAreaMessage | null | undefined if (!message?.attachments || message.attachments.length === 0) { return []; } - return message.attachments.map((item) => ({ + return message.attachments.map(toPendingAttachment); +} + +export function toPendingAttachment(item: MessageAttachment): PendingAttachment { + return { fileID: item.fileID, fileName: item.fileName, mimeType: item.mimeType, @@ -26,7 +30,7 @@ export function toPendingAttachments(message: ChatAreaMessage | null | undefined ragReady: item.ragReady, ragReason: item.ragReason, ocrUsed: item.ocrUsed, - })); + }; } export function resolvePersistedPublicID(value: string | null | undefined): string | null { diff --git a/frontend/i18n/messages/en-US/admin-channels.json b/frontend/i18n/messages/en-US/admin-channels.json index ca64d1d2..c69e8e27 100644 --- a/frontend/i18n/messages/en-US/admin-channels.json +++ b/frontend/i18n/messages/en-US/admin-channels.json @@ -92,7 +92,7 @@ }, "deleteDialog": { "title": "Delete upstream", - "description": "Delete upstream {name}? This removes the upstream and all route bindings, but keeps the model catalog.", + "description": "Delete upstream \"{name}\"? This removes the upstream and all route bindings, but keeps the model catalog.", "deleting": "Deleting...", "bulkTitle": "Bulk delete upstreams", "bulkDescription": "This will delete {count} upstreams and all route bindings, but keep model catalogs.", diff --git a/frontend/i18n/messages/en-US/admin-models.json b/frontend/i18n/messages/en-US/admin-models.json index ea7faa92..07b60363 100644 --- a/frontend/i18n/messages/en-US/admin-models.json +++ b/frontend/i18n/messages/en-US/admin-models.json @@ -49,7 +49,7 @@ }, "deleteDialog": { "title": "Delete model", - "description": "Delete model {model} and remove its bindings from all upstreams. This action cannot be undone.", + "description": "Delete model \"{model}\" and remove its bindings from all upstreams. This action cannot be undone.", "confirm": "Delete", "deleting": "Deleting", "bulkTitle": "Bulk delete models", @@ -82,7 +82,7 @@ "enableSource": "Enable source", "deleteSource": "Delete source", "deleteTitle": "Delete upstream source", - "deleteDescription": "Delete the route binding for upstream model {name}? This source will no longer be used for the current model.", + "deleteDescription": "Delete the route binding for upstream model \"{name}\"? This source will no longer be used for the current model.", "confirmDeleteSource": "Delete", "deletingSource": "Deleting", "circuitUntil": "Expected recovery: {time}", diff --git a/frontend/i18n/messages/en-US/chat.json b/frontend/i18n/messages/en-US/chat.json index 94a02472..d30cadc9 100644 --- a/frontend/i18n/messages/en-US/chat.json +++ b/frontend/i18n/messages/en-US/chat.json @@ -87,6 +87,7 @@ "footnoteBackref": "Back to footnote reference", "imageUnavailable": "Image not available", "previewImage": "Preview image", + "editImage": "Edit image", "downloadImage": "Download image", "externalLink": { "title": "Open external link?", @@ -184,6 +185,14 @@ "retrievingContent": "Retrieving relevant content...", "retryLater": "Please try again later.", "imagePreviewAlt": "Generated image preview", + "mediaInputUnsupported": "The selected model cannot process this input", + "mediaInputBlocked": { + "image_edit_input_required": "This model edits images. Upload an image to edit first.", + "image_edit_unsupported": "This model does not support image editing. Remove the image or select an editing-capable model.", + "image_generation_rejects_attachments": "Image generation accepts text prompts only. Remove attachments or select a model that handles them.", + "image_task_rejects_non_image_attachments": "Image tasks only support image attachments. Remove non-image files.", + "model_task_unsupported": "This model cannot process the current input. Select another model and try again." + }, "mediaStatus": { "queued": "Image task queued", "running": "Generating image", @@ -283,6 +292,23 @@ "booleanOn": "On", "booleanOff": "Off", "tool": "Tool {id}", + "mediaMode": { + "imageGeneration": "Image generation", + "imageGenerationIntro": "Image generation mode is active.", + "imageGenerationDescription": "Creates a new image from the text prompt. Upload an image to switch to image editing.", + "imageEdit": "Image editing", + "imageEditIntro": "Image editing mode is active.", + "imageEditDescription": "Edits the uploaded image using the text prompt.", + "invalidFile": "Invalid file detected", + "invalidFileIntro": "This file cannot be used for an image task.", + "blockedDescriptions": { + "image_edit_input_required": "Upload an image to edit first.", + "image_edit_unsupported": "The selected model does not support image editing. Remove the image or switch models.", + "image_generation_rejects_attachments": "Image generation only accepts text prompts. Remove attachments.", + "image_task_rejects_non_image_attachments": "Image tasks only support image attachments. Remove non-image files.", + "model_task_unsupported": "The selected model cannot process this input. Switch models and try again." + } + }, "nativeTools": { "grok": "Grok native tools", "openai": "OpenAI native tools", diff --git a/frontend/i18n/messages/en-US/errors.json b/frontend/i18n/messages/en-US/errors.json index 3ee2e060..dd44dc15 100644 --- a/frontend/i18n/messages/en-US/errors.json +++ b/frontend/i18n/messages/en-US/errors.json @@ -128,12 +128,13 @@ "llm": { "apiKeysRequired": "API keys are required.", "emptyResponse": "The model returned an empty response.", - "invalidAdapter": "Invalid model adapter.", + "invalidAdapter": "This interface protocol is not supported. Choose another protocol.", "invalidApiKeysConfig": "Invalid API keys configuration.", "invalidCompatible": "Invalid compatibility mode.", "invalidHeadersConfig": "Invalid headers configuration.", "invalidKinds": "Invalid model kinds.", "invalidPlatformModelName": "Invalid platform model name.", + "invalidRouteProtocolCombination": "This protocol combination is not available. A binding can use one protocol, or image generation and image editing protocols from the same provider.", "systemPromptTooLong": "System prompt must be at most 20000 characters.", "invalidProtocolDefaultsConfig": "Invalid protocol defaults configuration.", "modelRouteNotConfigured": "Model route is not configured.", @@ -142,7 +143,7 @@ "platformModelNameRequired": "Platform model name is required.", "protocolRequired": "Protocol is required.", "remoteModelsUnavailable": "Remote models are unavailable.", - "routeConflict": "This upstream is already bound to the same target model.", + "routeConflict": "This upstream model already has a binding for the same platform model and protocol. Remove the duplicate binding before saving.", "routesUnavailable": "All model routes are unavailable." }, "mcp": { @@ -165,6 +166,15 @@ "memory": { "keyRequired": "Memory key is required." }, + "media": { + "imagePromptRequired": "Enter an image prompt.", + "imageGenerationRejectsInputs": "Image generation does not accept input images. Remove attachments or choose image editing.", + "imageEditInputRequired": "Image editing requires at least one input image.", + "imageEditTooManyInputs": "Too many input images for image editing.", + "imageEditInputInvalid": "Image editing only supports valid image files.", + "routeProtocolMismatch": "The current model route protocol does not match this image task. Check the model binding.", + "invalidTask": "Invalid image task parameters." + }, "message": { "feedbackTargetInvalid": "Invalid message feedback target.", "invalidBranch": "Invalid message branch.", diff --git a/frontend/i18n/messages/zh-CN/admin-channels.json b/frontend/i18n/messages/zh-CN/admin-channels.json index 33231a7c..3d6de442 100644 --- a/frontend/i18n/messages/zh-CN/admin-channels.json +++ b/frontend/i18n/messages/zh-CN/admin-channels.json @@ -92,7 +92,7 @@ }, "deleteDialog": { "title": "删除上游", - "description": "确定要删除上游 {name} 吗？此操作会删除该上游及其全部路由绑定，但会保留模型目录。", + "description": "确定要删除上游「{name}」吗？此操作会删除该上游及其全部路由绑定，但会保留模型目录。", "deleting": "删除中…", "bulkTitle": "批量删除上游", "bulkDescription": "将删除 {count} 个上游及其全部路由绑定，但会保留模型目录。", diff --git a/frontend/i18n/messages/zh-CN/admin-models.json b/frontend/i18n/messages/zh-CN/admin-models.json index 7380a2bb..ca1e71ce 100644 --- a/frontend/i18n/messages/zh-CN/admin-models.json +++ b/frontend/i18n/messages/zh-CN/admin-models.json @@ -49,7 +49,7 @@ }, "deleteDialog": { "title": "确认删除模型", - "description": "将删除模型 {model}，并删除它在所有上游下的绑定关系，此操作不可撤销。", + "description": "将删除模型「{model}」，并删除它在所有上游下的绑定关系，此操作不可撤销。", "confirm": "确认删除", "deleting": "删除中", "bulkTitle": "确认批量删除模型", @@ -82,7 +82,7 @@ "enableSource": "启用来源", "deleteSource": "删除来源", "deleteTitle": "删除上游来源", - "deleteDescription": "确认删除上游模型 {name} 的路由绑定？删除后该来源将不再参与当前模型路由。", + "deleteDescription": "确认删除上游模型「{name}」的路由绑定？删除后该来源将不再参与当前模型路由。", "confirmDeleteSource": "删除", "deletingSource": "删除中", "circuitUntil": "预计恢复：{time}", diff --git a/frontend/i18n/messages/zh-CN/chat.json b/frontend/i18n/messages/zh-CN/chat.json index 336c2248..58207867 100644 --- a/frontend/i18n/messages/zh-CN/chat.json +++ b/frontend/i18n/messages/zh-CN/chat.json @@ -87,6 +87,7 @@ "footnoteBackref": "返回脚注引用", "imageUnavailable": "图片不可用", "previewImage": "放大图片", + "editImage": "编辑图片", "downloadImage": "下载图片", "externalLink": { "title": "打开外部链接？", @@ -184,6 +185,14 @@ "retrievingContent": "正在检索相关内容…", "retryLater": "请稍后重试。", "imagePreviewAlt": "生成图片预览", + "mediaInputUnsupported": "当前输入无法使用所选模型", + "mediaInputBlocked": { + "image_edit_input_required": "该模型用于图像编辑，请先上传需要编辑的图片。", + "image_edit_unsupported": "该模型不支持图像编辑，请移除图片或切换到支持编辑的模型。", + "image_generation_rejects_attachments": "图像生成仅接受文字提示词，请移除附件或切换到可处理附件的模型。", + "image_task_rejects_non_image_attachments": "图像任务仅支持图片附件，请移除非图片文件。", + "model_task_unsupported": "该模型无法处理当前输入，请切换模型后重试。" + }, "mediaStatus": { "queued": "图片任务已排队", "running": "正在生成图片", @@ -283,6 +292,23 @@ "booleanOn": "开启", "booleanOff": "关闭", "tool": "工具 {id}", + "mediaMode": { + "imageGeneration": "图像生成", + "imageGenerationIntro": "当前为图像生成模式。", + "imageGenerationDescription": "将根据文字提示创建新图像；上传图片后会自动切换为图像编辑。", + "imageEdit": "图像编辑", + "imageEditIntro": "当前为图像编辑模式。", + "imageEditDescription": "将基于已上传图片和文字提示进行编辑。", + "invalidFile": "监测到无效文件", + "invalidFileIntro": "当前文件无法用于图像任务。", + "blockedDescriptions": { + "image_edit_input_required": "请先上传需要编辑的图片。", + "image_edit_unsupported": "当前模型不支持图像编辑，请移除图片或切换模型。", + "image_generation_rejects_attachments": "图像生成仅接收文字提示，请移除附件。", + "image_task_rejects_non_image_attachments": "图像任务仅支持图片附件，请移除非图片文件。", + "model_task_unsupported": "当前模型不支持此输入，请切换模型后重试。" + } + }, "nativeTools": { "grok": "Grok 原生工具", "openai": "OpenAI 原生工具", diff --git a/frontend/i18n/messages/zh-CN/errors.json b/frontend/i18n/messages/zh-CN/errors.json index 0bfd7de1..48789787 100644 --- a/frontend/i18n/messages/zh-CN/errors.json +++ b/frontend/i18n/messages/zh-CN/errors.json @@ -128,12 +128,13 @@ "llm": { "apiKeysRequired": "API Keys 不能为空。", "emptyResponse": "模型返回了空响应。", - "invalidAdapter": "模型适配器无效。", + "invalidAdapter": "接口协议不受支持，请重新选择协议。", "invalidApiKeysConfig": "API Keys 配置无效。", "invalidCompatible": "兼容模式无效。", "invalidHeadersConfig": "请求头配置无效。", "invalidKinds": "模型类型无效。", "invalidPlatformModelName": "平台模型名称无效。", + "invalidRouteProtocolCombination": "当前协议组合不可用。同一绑定只能选择单个协议，或同时选择同一厂商的图像生成与图像编辑协议。", "systemPromptTooLong": "系统提示词不能超过 20000 个字符。", "invalidProtocolDefaultsConfig": "协议默认配置无效。", "modelRouteNotConfigured": "模型路由未配置。", @@ -142,7 +143,7 @@ "platformModelNameRequired": "平台模型名称不能为空。", "protocolRequired": "协议不能为空。", "remoteModelsUnavailable": "远程模型列表暂不可用。", - "routeConflict": "该上游已绑定相同目标模型。", + "routeConflict": "该上游模型已存在相同平台模型和协议的绑定，请删除重复绑定后再保存。", "routesUnavailable": "所有模型路由都不可用。" }, "mcp": { @@ -165,6 +166,15 @@ "memory": { "keyRequired": "记忆键不能为空。" }, + "media": { + "imagePromptRequired": "请输入图片提示词。", + "imageGenerationRejectsInputs": "图片生成不支持携带输入图片，请移除附件或选择图片编辑。", + "imageEditInputRequired": "图片编辑需要至少上传一张输入图片。", + "imageEditTooManyInputs": "图片编辑输入图片过多。", + "imageEditInputInvalid": "图片编辑只支持有效的图片文件。", + "routeProtocolMismatch": "当前模型路由协议与图片任务不匹配，请检查模型绑定。", + "invalidTask": "图片任务参数无效。" + }, "message": { "feedbackTargetInvalid": "消息反馈目标无效。", "invalidBranch": "消息分支无效。", diff --git a/frontend/i18n/resolve-error-message.ts b/frontend/i18n/resolve-error-message.ts index 90d5098d..469df465 100644 --- a/frontend/i18n/resolve-error-message.ts +++ b/frontend/i18n/resolve-error-message.ts @@ -39,6 +39,7 @@ const REQUEST_FIELD_LABELS: Record> = { email: "Email", headersJSON: "Headers JSON", locale: "Language", + maskFileID: "Mask file", name: "Name", password: "Password", phone: "Phone", @@ -67,6 +68,7 @@ const REQUEST_FIELD_LABELS: Record> = { email: "邮箱", headersJSON: "请求头 JSON", locale: "语言", + maskFileID: "蒙版文件", name: "名称", password: "密码", phone: "手机号", diff --git a/frontend/next-env.d.ts b/frontend/next-env.d.ts index c4b7818f..9edff1c7 100644 --- a/frontend/next-env.d.ts +++ b/frontend/next-env.d.ts @@ -1,6 +1,6 @@ /// /// -import "./.next/dev/types/routes.d.ts"; +import "./.next/types/routes.d.ts"; // NOTE: This file should not be edited // see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/frontend/shared/api/conversation.types.ts b/frontend/shared/api/conversation.types.ts index d7e4bb40..f0c8cec8 100644 --- a/frontend/shared/api/conversation.types.ts +++ b/frontend/shared/api/conversation.types.ts @@ -380,6 +380,7 @@ export type MediaImageRequest = { options?: ConversationOptions; clientRunID?: string; fileIDs?: string[]; + maskFileID?: string; parentMessagePublicID?: string; sourceMessagePublicID?: string; branchReason?: "default" | "retry" | "edit"; diff --git a/frontend/shared/lib/model-option-policy.ts b/frontend/shared/lib/model-option-policy.ts index f68ab876..a935fb59 100644 --- a/frontend/shared/lib/model-option-policy.ts +++ b/frontend/shared/lib/model-option-policy.ts @@ -3,11 +3,13 @@ export const MODEL_OPTION_POLICY_PROTOCOLS = [ "openai_chat_completions", "openai_responses", "openai_image_generations", + "openai_image_edits", "anthropic_messages", "gemini_generate_content", "google_image_generation", "xai_responses", "xai_image", + "xai_image_edits", ] as const; export type ModelOptionPolicyProtocol = (typeof MODEL_OPTION_POLICY_PROTOCOLS)[number]; @@ -58,12 +60,14 @@ export const MODEL_OPTION_POLICY_PROTOCOL_LABELS: Record