Skip to content

Commit cdacfd7

Browse files
chenymechenyme
authored
Merge pull request #26 from DEEIX-AI/injection_setting
feat: add full-text injection limits configuration and validation
2 parents fb4d9e5 + cae655c commit cdacfd7

9 files changed

Lines changed: 263 additions & 33 deletions

File tree

‎backend/internal/application/settings/runtime_settings.go‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -152,16 +152,18 @@ func (r *RuntimeSettings) applyItem(cfg *config.Config, item domainsettings.Syst
152152
// 文件处理配置
153153
case "file:image_max_dimension":
154154
cfg.ImageMaxDimension = toInt(item.Value, cfg.ImageMaxDimension)
155+
case "file:full_context_limit_enabled":
156+
cfg.FileFullContextLimitEnabled = toBool(item.Value, cfg.FileFullContextLimitEnabled)
155157
case "file:file_full_context_max_bytes":
156-
cfg.FileFullContextMaxBytes = toInt64(item.Value, cfg.FileFullContextMaxBytes)
158+
cfg.FileFullContextMaxBytes = toOptionalInt64(item.Value, cfg.FileFullContextMaxBytes)
157159
case "file:full_context_max_tokens":
158-
cfg.FileFullContextMaxTokens = toInt(item.Value, cfg.FileFullContextMaxTokens)
160+
cfg.FileFullContextMaxTokens = toOptionalInt(item.Value, cfg.FileFullContextMaxTokens)
159161
case "file:image_max_bytes":
160162
cfg.FileImageMaxBytes = toOptionalInt64(item.Value, cfg.FileImageMaxBytes)
161163
case "file:doc_max_bytes":
162164
cfg.FileDocMaxBytes = toOptionalInt64(item.Value, cfg.FileDocMaxBytes)
163165
case "file:full_context_pdf_max_pages":
164-
cfg.FileFullContextPDFMaxPages = toInt(item.Value, cfg.FileFullContextPDFMaxPages)
166+
cfg.FileFullContextPDFMaxPages = toOptionalInt(item.Value, cfg.FileFullContextPDFMaxPages)
165167
case "file:allowed_mime_types":
166168
cfg.FileAllowedMIMETypes = item.Value
167169
case "extract:engine":
@@ -368,18 +370,30 @@ func (r *RuntimeSettings) normalizeConfig(cfg *config.Config) {
368370
if strings.TrimSpace(cfg.NativeToolAllowedTypes) == "" {
369371
cfg.NativeToolAllowedTypes = config.DefaultNativeToolAllowedTypesJSON()
370372
}
373+
if !cfg.FileFullContextLimitEnabled {
374+
cfg.FileFullContextMaxBytes = 0
375+
cfg.FileFullContextMaxTokens = 0
376+
cfg.FileFullContextPDFMaxPages = 0
377+
}
371378
}
372379

373380
func toInt(s string, fallback int) int {
374-
v, err := strconv.Atoi(s)
381+
v, err := strconv.Atoi(strings.TrimSpace(s))
375382
if err != nil {
376383
return fallback
377384
}
378385
return v
379386
}
380387

388+
func toOptionalInt(s string, fallback int) int {
389+
if strings.TrimSpace(s) == "" {
390+
return 0
391+
}
392+
return toInt(s, fallback)
393+
}
394+
381395
func toInt64(s string, fallback int64) int64 {
382-
v, err := strconv.ParseInt(s, 10, 64)
396+
v, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64)
383397
if err != nil {
384398
return fallback
385399
}
@@ -402,7 +416,7 @@ func toBool(s string, fallback bool) bool {
402416
}
403417

404418
func toFloat(s string, fallback float64) float64 {
405-
v, err := strconv.ParseFloat(s, 64)
419+
v, err := strconv.ParseFloat(strings.TrimSpace(s), 64)
406420
if err != nil {
407421
return fallback
408422
}

‎backend/internal/application/settings/seed.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -71,11 +71,12 @@ func defaultSettings() []domainsettings.SystemSetting {
7171

7272
// 文件处理配置
7373
{Namespace: "file", Key: "image_max_dimension", Value: "1024", ValueType: "int", Description: "图片发送前缩放最大边长(px)，0=不缩放"},
74-
{Namespace: "file", Key: "file_full_context_max_bytes", Value: "51200", ValueType: "int", Description: "文本文件全文注入最大字节数(50KB)"},
75-
{Namespace: "file", Key: "full_context_max_tokens", Value: "12000", ValueType: "int", Description: "全文注入最大token预算"},
74+
{Namespace: "file", Key: "full_context_limit_enabled", Value: "true", ValueType: "bool", Description: "是否启用全文注入大小、Token、PDF页数限制"},
75+
{Namespace: "file", Key: "file_full_context_max_bytes", Value: "51200", ValueType: "int", Description: "文本文件全文注入最大字节数(50KB)，留空或0表示不限制"},
76+
{Namespace: "file", Key: "full_context_max_tokens", Value: "12000", ValueType: "int", Description: "全文注入最大token预算，留空或0表示不限制"},
7677
{Namespace: "file", Key: "image_max_bytes", Value: "", ValueType: "int", Description: "图片单文件大小上限(字节)，留空则回退默认附件大小上限"},
7778
{Namespace: "file", Key: "doc_max_bytes", Value: "", ValueType: "int", Description: "文档单文件大小上限(字节)，留空则回退默认附件大小上限"},
78-
{Namespace: "file", Key: "full_context_pdf_max_pages", Value: "20", ValueType: "int", Description: "PDF Full Context最大页数，超出走RAG"},
79+
{Namespace: "file", Key: "full_context_pdf_max_pages", Value: "20", ValueType: "int", Description: "PDF Full Context最大页数，留空或0表示不限制"},
7980
{Namespace: "file", Key: "allowed_mime_types", Value: defaultAllowedMIMETypes, ValueType: "string", Description: "白名单MIME类型(逗号分隔)"},
8081
{Namespace: "extract", Key: "engine", Value: "builtin", ValueType: "string", Description: "提取主引擎枚举(builtin/tika/docling/mineru)"},
8182
{Namespace: "extract", Key: "ocr_engine", Value: "rapidocr", ValueType: "string", Description: "OCR 引擎枚举(rapidocr/tesseract/paddle/tencent/aliyun/llm)"},

‎backend/internal/application/settings/service.go‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -274,8 +274,10 @@ func validatePatchItem(item PatchItem) error {
274274
return validateIntMinMax(value, 1, 65535, key)
275275
case "auth:email_registration_allowed_domains":
276276
return validateEmailDomainList(value, key)
277-
case "storage:max_upload_file_bytes", "storage:user_storage_quota_bytes", "file:file_full_context_max_bytes":
277+
case "storage:max_upload_file_bytes", "storage:user_storage_quota_bytes":
278278
return validateInt64Min(value, 1, key)
279+
case "file:file_full_context_max_bytes":
280+
return validateOptionalInt64Min(value, 0, key)
279281
case "file:image_max_bytes", "file:doc_max_bytes":
280282
if value == "" {
281283
return nil
@@ -286,9 +288,9 @@ func validatePatchItem(item PatchItem) error {
286288
case "file:image_max_dimension":
287289
return validateIntMinMax(value, 0, 8192, key)
288290
case "file:full_context_max_tokens":
289-
return validateIntMinMax(value, 128, 1000000, key)
291+
return validateOptionalIntZeroOrMinMax(value, 128, 1000000, key)
290292
case "file:full_context_pdf_max_pages":
291-
return validateIntMinMax(value, 1, 500, key)
293+
return validateOptionalIntZeroOrMinMax(value, 1, 500, key)
292294
case "chat:rag_wait_ready_ms":
293295
return validateIntMinMax(value, 1000, 120000, key)
294296
case "chat:context_artifact_retention_days":
@@ -372,7 +374,7 @@ func validatePatchItem(item PatchItem) error {
372374
return validateStringMax(value, 255, key)
373375
case "extract:tencent_ocr_secret_id", "extract:tencent_ocr_secret_key", "extract:aliyun_ocr_access_key_id", "extract:aliyun_ocr_access_key_secret":
374376
return validateStringMax(value, 512, key)
375-
case "auth:username_login_enabled", "auth:email_login_enabled", "auth:third_party_login_enabled", "auth:email_registration_enabled", "auth:email_verification_enabled", "auth:email_registration_block_plus_alias", "auth:auto_link_verified_email", "chat:rag_enabled", "chat:message_embedding_enabled", "chat:semantic_context_enabled", "file:embedding_enabled", "file:embed_trigger_on_upload", "file:embedding_normalize", "extract:image_ocr_enabled", "extract:pdf_ocr_fallback_enabled", "mcp:mcp_enable":
377+
case "auth:username_login_enabled", "auth:email_login_enabled", "auth:third_party_login_enabled", "auth:email_registration_enabled", "auth:email_verification_enabled", "auth:email_registration_block_plus_alias", "auth:auto_link_verified_email", "chat:rag_enabled", "chat:message_embedding_enabled", "chat:semantic_context_enabled", "file:full_context_limit_enabled", "file:embedding_enabled", "file:embed_trigger_on_upload", "file:embedding_normalize", "extract:image_ocr_enabled", "extract:pdf_ocr_fallback_enabled", "mcp:mcp_enable":
376378
if _, err := strconv.ParseBool(value); err != nil {
377379
return fmt.Errorf("%s must be bool", key)
378380
}
@@ -814,6 +816,28 @@ func validateInt64Min(value string, min int64, key string) error {
814816
return nil
815817
}
816818

819+
func validateOptionalInt64Min(value string, min int64, key string) error {
820+
if strings.TrimSpace(value) == "" {
821+
return nil
822+
}
823+
v, err := strconv.ParseInt(value, 10, 64)
824+
if err != nil || v < min {
825+
return fmt.Errorf("%s must be empty or >= %d", key, min)
826+
}
827+
return nil
828+
}
829+
830+
func validateOptionalIntZeroOrMinMax(value string, min int, max int, key string) error {
831+
if strings.TrimSpace(value) == "" {
832+
return nil
833+
}
834+
v, err := strconv.Atoi(value)
835+
if err != nil || v < 0 || (v > 0 && (v < min || v > max)) {
836+
return fmt.Errorf("%s must be empty, 0, or between %d and %d", key, min, max)
837+
}
838+
return nil
839+
}
840+
817841
func validateFloatMinMax(value string, min float64, max float64, key string) error {
818842
v, err := strconv.ParseFloat(strings.TrimSpace(value), 64)
819843
if err != nil || v < min || v > max {

‎backend/internal/application/settings/service_test.go‎

Lines changed: 114 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -135,11 +135,120 @@ func TestValidateModelOptionPolicySettings(t *testing.T) {
135135
}
136136
}
137137

138-
func TestValidateFullContextMaxTokensAllowsLargeContextWindows(t *testing.T) {
139-
if err := validatePatchItem(PatchItem{Namespace: "file", Key: "full_context_max_tokens", Value: "1000000"}); err != nil {
140-
t.Fatalf("expected 1M full context token limit to pass, got %v", err)
138+
func TestValidateFullContextLimitsAllowUnlimitedValues(t *testing.T) {
139+
cases := []PatchItem{
140+
{Namespace: "file", Key: "full_context_limit_enabled", Value: "true"},
141+
{Namespace: "file", Key: "full_context_limit_enabled", Value: "false"},
142+
{Namespace: "file", Key: "file_full_context_max_bytes", Value: ""},
143+
{Namespace: "file", Key: "file_full_context_max_bytes", Value: "0"},
144+
{Namespace: "file", Key: "full_context_max_tokens", Value: ""},
145+
{Namespace: "file", Key: "full_context_max_tokens", Value: "0"},
146+
{Namespace: "file", Key: "full_context_pdf_max_pages", Value: ""},
147+
{Namespace: "file", Key: "full_context_pdf_max_pages", Value: "0"},
148+
}
149+
150+
for _, item := range cases {
151+
if err := validatePatchItem(item); err != nil {
152+
t.Fatalf("expected %s:%s=%q to pass, got %v", item.Namespace, item.Key, item.Value, err)
153+
}
154+
}
155+
}
156+
157+
func TestValidateFullContextLimitsEnforcesConfiguredRanges(t *testing.T) {
158+
cases := []struct {
159+
name string
160+
item PatchItem
161+
want bool
162+
}{
163+
{
164+
name: "full context limit mode must be boolean",
165+
item: PatchItem{Namespace: "file", Key: "full_context_limit_enabled", Value: "disabled"},
166+
want: false,
167+
},
168+
{
169+
name: "1M token limit passes",
170+
item: PatchItem{Namespace: "file", Key: "full_context_max_tokens", Value: "1000000"},
171+
want: true,
172+
},
173+
{
174+
name: "token limit below minimum fails",
175+
item: PatchItem{Namespace: "file", Key: "full_context_max_tokens", Value: "127"},
176+
want: false,
177+
},
178+
{
179+
name: "token limit above maximum fails",
180+
item: PatchItem{Namespace: "file", Key: "full_context_max_tokens", Value: "1000001"},
181+
want: false,
182+
},
183+
{
184+
name: "negative byte limit fails",
185+
item: PatchItem{Namespace: "file", Key: "file_full_context_max_bytes", Value: "-1"},
186+
want: false,
187+
},
188+
{
189+
name: "pdf page limit above maximum fails",
190+
item: PatchItem{Namespace: "file", Key: "full_context_pdf_max_pages", Value: "501"},
191+
want: false,
192+
},
193+
}
194+
195+
for _, tc := range cases {
196+
t.Run(tc.name, func(t *testing.T) {
197+
err := validatePatchItem(tc.item)
198+
if tc.want && err != nil {
199+
t.Fatalf("expected validation to pass, got %v", err)
200+
}
201+
if !tc.want && err == nil {
202+
t.Fatal("expected validation to fail")
203+
}
204+
})
205+
}
206+
}
207+
208+
func TestRuntimeSettingsDisablesFullContextLimits(t *testing.T) {
209+
runtimeSettings := NewRuntimeSettings(nil, nil, "test-data-encryption-key")
210+
cfg := config.Config{
211+
FileFullContextLimitEnabled: true,
212+
FileFullContextMaxBytes: 51200,
213+
FileFullContextMaxTokens: 12000,
214+
FileFullContextPDFMaxPages: 20,
141215
}
142-
if err := validatePatchItem(PatchItem{Namespace: "file", Key: "full_context_max_tokens", Value: "1000001"}); err == nil {
143-
t.Fatal("expected full context token limit above 1M to fail")
216+
217+
runtimeSettings.applyItem(&cfg, domainsettings.SystemSetting{Namespace: "file", Key: "full_context_limit_enabled", Value: "false"})
218+
runtimeSettings.normalizeConfig(&cfg)
219+
220+
if cfg.FileFullContextLimitEnabled {
221+
t.Fatal("expected full context limit switch to be disabled")
222+
}
223+
if cfg.FileFullContextMaxBytes != 0 || cfg.FileFullContextMaxTokens != 0 || cfg.FileFullContextPDFMaxPages != 0 {
224+
t.Fatalf(
225+
"expected disabled full context limits to be unlimited, got bytes=%d tokens=%d pdfPages=%d",
226+
cfg.FileFullContextMaxBytes,
227+
cfg.FileFullContextMaxTokens,
228+
cfg.FileFullContextPDFMaxPages,
229+
)
230+
}
231+
}
232+
233+
func TestRuntimeSettingsTreatsEmptyFullContextLimitsAsUnlimited(t *testing.T) {
234+
runtimeSettings := NewRuntimeSettings(nil, nil, "test-data-encryption-key")
235+
cfg := config.Config{
236+
FileFullContextLimitEnabled: true,
237+
FileFullContextMaxBytes: 51200,
238+
FileFullContextMaxTokens: 12000,
239+
FileFullContextPDFMaxPages: 20,
240+
}
241+
242+
runtimeSettings.applyItem(&cfg, domainsettings.SystemSetting{Namespace: "file", Key: "file_full_context_max_bytes", Value: ""})
243+
runtimeSettings.applyItem(&cfg, domainsettings.SystemSetting{Namespace: "file", Key: "full_context_max_tokens", Value: ""})
244+
runtimeSettings.applyItem(&cfg, domainsettings.SystemSetting{Namespace: "file", Key: "full_context_pdf_max_pages", Value: ""})
245+
246+
if cfg.FileFullContextMaxBytes != 0 || cfg.FileFullContextMaxTokens != 0 || cfg.FileFullContextPDFMaxPages != 0 {
247+
t.Fatalf(
248+
"expected empty full context limits to be unlimited, got bytes=%d tokens=%d pdfPages=%d",
249+
cfg.FileFullContextMaxBytes,
250+
cfg.FileFullContextMaxTokens,
251+
cfg.FileFullContextPDFMaxPages,
252+
)
144253
}
145254
}

‎backend/internal/infra/config/config.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -325,6 +325,7 @@ type Config struct {
325325
MaxMessageFiles int
326326
// 文件处理配置
327327
ImageMaxDimension int // 图片缩放最大边长（像素），0 = 不缩放
328+
FileFullContextLimitEnabled bool // 是否启用全文注入阈值限制
328329
FileFullContextMaxBytes int64 // 文本文件全文注入阈值（字节），超出不注入
329330
FileFullContextMaxTokens int // 文本文件全文注入阈值（token）
330331
FileImageMaxBytes int64 // 图片单文件上限（字节）
@@ -518,6 +519,7 @@ func Load() Config {
518519
MaxUploadFileBytes: 20971520,
519520
MaxMessageFiles: 10,
520521
ImageMaxDimension: 1024,
522+
FileFullContextLimitEnabled: true,
521523
FileFullContextMaxBytes: 51200, // 50KB
522524
FileFullContextMaxTokens: 12000,
523525
FileImageMaxBytes: 0,

‎frontend/features/admin/model/chat-files.ts‎

Lines changed: 46 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,11 @@ export const EMBEDDING_MODES = {
8989
ON: "true",
9090
} as const;
9191

92+
export const FULL_CONTEXT_LIMIT_MODES = {
93+
OFF: "false",
94+
ON: "true",
95+
} as const;
96+
9297
export const MINERU_SERVICE_SOURCES = {
9398
CLOUD: "cloud",
9499
SELF_HOSTED: "self_hosted",
@@ -593,9 +598,44 @@ export const SETTINGS_GROUPS: SettingsGroup[] = [
593598
title: "Full-text injection",
594599
description: "Controls whether extracted file text can be injected directly into context instead of using RAG.",
595600
fields: [
596-
{ namespace: "file", key: "file_full_context_max_bytes", label: "Full-text byte limit", description: "Maximum extracted text bytes allowed for full-context injection.", type: "int", placeholder: "Byte limit" },
597-
{ namespace: "file", key: "full_context_max_tokens", label: "Full-text token limit", description: "Maximum token budget allowed for full-context injection.", type: "int", placeholder: "Token limit" },
598-
{ namespace: "file", key: "full_context_pdf_max_pages", label: "Full-text page limit", description: "Maximum PDF pages allowed for full-context injection.", type: "int", placeholder: "Max pages" },
601+
{
602+
namespace: "file",
603+
key: "full_context_limit_enabled",
604+
label: "Full-text injection limits",
605+
description: "Enable byte, token, and PDF page limits for full-text injection.",
606+
type: "select",
607+
options: [
608+
{ label: "Off", value: FULL_CONTEXT_LIMIT_MODES.OFF },
609+
{ label: "On", value: FULL_CONTEXT_LIMIT_MODES.ON },
610+
],
611+
},
612+
{
613+
namespace: "file",
614+
key: "file_full_context_max_bytes",
615+
label: "Full-text byte limit",
616+
description: "Maximum extracted text bytes allowed for full-context injection. Leave empty or use 0 for no byte limit.",
617+
type: "int",
618+
placeholder: "Empty or 0 means unlimited",
619+
visibleWhen: { field: "file.full_context_limit_enabled", equals: FULL_CONTEXT_LIMIT_MODES.ON },
620+
},
621+
{
622+
namespace: "file",
623+
key: "full_context_max_tokens",
624+
label: "Full-text token limit",
625+
description: "Maximum token budget allowed for full-context injection. Leave empty or use 0 for no token limit.",
626+
type: "int",
627+
placeholder: "Empty or 0 means unlimited",
628+
visibleWhen: { field: "file.full_context_limit_enabled", equals: FULL_CONTEXT_LIMIT_MODES.ON },
629+
},
630+
{
631+
namespace: "file",
632+
key: "full_context_pdf_max_pages",
633+
label: "Full-text page limit",
634+
description: "Maximum PDF pages allowed for full-context injection. Leave empty or use 0 for no page limit.",
635+
type: "int",
636+
placeholder: "Empty or 0 means unlimited",
637+
visibleWhen: { field: "file.full_context_limit_enabled", equals: FULL_CONTEXT_LIMIT_MODES.ON },
638+
},
599639
],
600640
},
601641
{
@@ -1065,6 +1105,9 @@ export function applySettingsDefaults(next: Record<string, string>): Record<stri
10651105
if (!["true", "false"].includes(result["file.embedding_enabled"] ?? "")) {
10661106
result["file.embedding_enabled"] = EMBEDDING_MODES.OFF;
10671107
}
1108+
if (!["true", "false"].includes(result["file.full_context_limit_enabled"] ?? "")) {
1109+
result["file.full_context_limit_enabled"] = FULL_CONTEXT_LIMIT_MODES.ON;
1110+
}
10681111
if (!["true", "false"].includes(result["chat.rag_enabled"] ?? "")) {
10691112
result["chat.rag_enabled"] = "false";
10701113
}

‎frontend/i18n/messages/en-US/admin-files.json‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -102,20 +102,28 @@
102102
"description": "Override the size limit for document attachments. Falls back to the default attachment limit when empty.",
103103
"placeholder": "Leave empty to use the default attachment limit"
104104
},
105+
"full_context_limit_enabled": {
106+
"label": "Full-text injection limits",
107+
"description": "When enabled, full-text injection is limited by byte size, token budget, and PDF pages. When disabled, these limits are ignored.",
108+
"options": {
109+
"false": "Off",
110+
"true": "On"
111+
}
112+
},
105113
"file_full_context_max_bytes": {
106114
"label": "Full-text byte limit",
107-
"description": "Maximum extracted text bytes allowed for full-context injection.",
108-
"placeholder": "Byte limit"
115+
"description": "Maximum extracted text bytes allowed for full-context injection. Leave empty or use 0 for no byte limit.",
116+
"placeholder": "Empty or 0 means unlimited"
109117
},
110118
"full_context_max_tokens": {
111119
"label": "Full-text token limit",
112-
"description": "Maximum token budget allowed for full-context injection.",
113-
"placeholder": "Token limit"
120+
"description": "Maximum token budget allowed for full-context injection. Leave empty or use 0 for no token limit.",
121+
"placeholder": "Empty or 0 means unlimited"
114122
},
115123
"full_context_pdf_max_pages": {
116124
"label": "Full-text page limit",
117-
"description": "Maximum PDF pages allowed for full-context injection.",
118-
"placeholder": "Max pages"
125+
"description": "Maximum PDF pages allowed for full-context injection. Leave empty or use 0 for no page limit.",
126+
"placeholder": "Empty or 0 means unlimited"
119127
},
120128
"embedding_enabled": {
121129
"label": "Embedding",

0 commit comments

Comments
 (0)