Skip to content

feat: add full-text injection limits configuration and validation #26

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chenyme merged 1 commit into from
May 23, 2026
Merged

feat: add full-text injection limits configuration and validation #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 20 additions & 6 deletions backend/internal/application/settings/runtime_settings.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,16 +152,18 @@ func (r *RuntimeSettings) applyItem(cfg *config.Config, item domainsettings.Syst
// 文件处理配置
case "file:image_max_dimension":
cfg.ImageMaxDimension = toInt(item.Value, cfg.ImageMaxDimension)
case "file:full_context_limit_enabled":
cfg.FileFullContextLimitEnabled = toBool(item.Value, cfg.FileFullContextLimitEnabled)
case "file:file_full_context_max_bytes":
cfg.FileFullContextMaxBytes = toInt64(item.Value, cfg.FileFullContextMaxBytes)
cfg.FileFullContextMaxBytes = toOptionalInt64(item.Value, cfg.FileFullContextMaxBytes)
case "file:full_context_max_tokens":
cfg.FileFullContextMaxTokens = toInt(item.Value, cfg.FileFullContextMaxTokens)
cfg.FileFullContextMaxTokens = toOptionalInt(item.Value, cfg.FileFullContextMaxTokens)
case "file:image_max_bytes":
cfg.FileImageMaxBytes = toOptionalInt64(item.Value, cfg.FileImageMaxBytes)
case "file:doc_max_bytes":
cfg.FileDocMaxBytes = toOptionalInt64(item.Value, cfg.FileDocMaxBytes)
case "file:full_context_pdf_max_pages":
cfg.FileFullContextPDFMaxPages = toInt(item.Value, cfg.FileFullContextPDFMaxPages)
cfg.FileFullContextPDFMaxPages = toOptionalInt(item.Value, cfg.FileFullContextPDFMaxPages)
case "file:allowed_mime_types":
cfg.FileAllowedMIMETypes = item.Value
case "extract:engine":
Expand Down Expand Up @@ -368,18 +370,30 @@ func (r *RuntimeSettings) normalizeConfig(cfg *config.Config) {
if strings.TrimSpace(cfg.NativeToolAllowedTypes) == "" {
cfg.NativeToolAllowedTypes = config.DefaultNativeToolAllowedTypesJSON()
}
if !cfg.FileFullContextLimitEnabled {
cfg.FileFullContextMaxBytes = 0
cfg.FileFullContextMaxTokens = 0
cfg.FileFullContextPDFMaxPages = 0
}
}

func toInt(s string, fallback int) int {
v, err := strconv.Atoi(s)
v, err := strconv.Atoi(strings.TrimSpace(s))
if err != nil {
return fallback
}
return v
}

func toOptionalInt(s string, fallback int) int {
if strings.TrimSpace(s) == "" {
return 0
}
return toInt(s, fallback)
}

func toInt64(s string, fallback int64) int64 {
v, err := strconv.ParseInt(s, 10, 64)
v, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64)
if err != nil {
return fallback
}
Expand All @@ -402,7 +416,7 @@ func toBool(s string, fallback bool) bool {
}

func toFloat(s string, fallback float64) float64 {
v, err := strconv.ParseFloat(s, 64)
v, err := strconv.ParseFloat(strings.TrimSpace(s), 64)
if err != nil {
return fallback
}
Expand Down
7 changes: 4 additions & 3 deletions backend/internal/application/settings/seed.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,11 +71,12 @@ func defaultSettings() []domainsettings.SystemSetting {

// 文件处理配置
{Namespace: "file", Key: "image_max_dimension", Value: "1024", ValueType: "int", Description: "图片发送前缩放最大边长(px)，0=不缩放"},
{Namespace: "file", Key: "file_full_context_max_bytes", Value: "51200", ValueType: "int", Description: "文本文件全文注入最大字节数(50KB)"},
{Namespace: "file", Key: "full_context_max_tokens", Value: "12000", ValueType: "int", Description: "全文注入最大token预算"},
{Namespace: "file", Key: "full_context_limit_enabled", Value: "true", ValueType: "bool", Description: "是否启用全文注入大小、Token、PDF页数限制"},
{Namespace: "file", Key: "file_full_context_max_bytes", Value: "51200", ValueType: "int", Description: "文本文件全文注入最大字节数(50KB)，留空或0表示不限制"},
{Namespace: "file", Key: "full_context_max_tokens", Value: "12000", ValueType: "int", Description: "全文注入最大token预算，留空或0表示不限制"},
{Namespace: "file", Key: "image_max_bytes", Value: "", ValueType: "int", Description: "图片单文件大小上限(字节)，留空则回退默认附件大小上限"},
{Namespace: "file", Key: "doc_max_bytes", Value: "", ValueType: "int", Description: "文档单文件大小上限(字节)，留空则回退默认附件大小上限"},
{Namespace: "file", Key: "full_context_pdf_max_pages", Value: "20", ValueType: "int", Description: "PDF Full Context最大页数，超出走RAG"},
{Namespace: "file", Key: "full_context_pdf_max_pages", Value: "20", ValueType: "int", Description: "PDF Full Context最大页数，留空或0表示不限制"},
{Namespace: "file", Key: "allowed_mime_types", Value: defaultAllowedMIMETypes, ValueType: "string", Description: "白名单MIME类型(逗号分隔)"},
{Namespace: "extract", Key: "engine", Value: "builtin", ValueType: "string", Description: "提取主引擎枚举(builtin/tika/docling/mineru)"},
{Namespace: "extract", Key: "ocr_engine", Value: "rapidocr", ValueType: "string", Description: "OCR 引擎枚举(rapidocr/tesseract/paddle/tencent/aliyun/llm)"},
Expand Down
32 changes: 28 additions & 4 deletions backend/internal/application/settings/service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -274,8 +274,10 @@ func validatePatchItem(item PatchItem) error {
return validateIntMinMax(value, 1, 65535, key)
case "auth:email_registration_allowed_domains":
return validateEmailDomainList(value, key)
case "storage:max_upload_file_bytes", "storage:user_storage_quota_bytes", "file:file_full_context_max_bytes":
case "storage:max_upload_file_bytes", "storage:user_storage_quota_bytes":
return validateInt64Min(value, 1, key)
case "file:file_full_context_max_bytes":
return validateOptionalInt64Min(value, 0, key)
case "file:image_max_bytes", "file:doc_max_bytes":
if value == "" {
return nil
Expand All @@ -286,9 +288,9 @@ func validatePatchItem(item PatchItem) error {
case "file:image_max_dimension":
return validateIntMinMax(value, 0, 8192, key)
case "file:full_context_max_tokens":
return validateIntMinMax(value, 128, 1000000, key)
return validateOptionalIntZeroOrMinMax(value, 128, 1000000, key)
case "file:full_context_pdf_max_pages":
return validateIntMinMax(value, 1, 500, key)
return validateOptionalIntZeroOrMinMax(value, 1, 500, key)
case "chat:rag_wait_ready_ms":
return validateIntMinMax(value, 1000, 120000, key)
case "chat:context_artifact_retention_days":
Expand Down Expand Up @@ -372,7 +374,7 @@ func validatePatchItem(item PatchItem) error {
return validateStringMax(value, 255, key)
case "extract:tencent_ocr_secret_id", "extract:tencent_ocr_secret_key", "extract:aliyun_ocr_access_key_id", "extract:aliyun_ocr_access_key_secret":
return validateStringMax(value, 512, key)
case "auth:username_login_enabled", "auth:email_login_enabled", "auth:third_party_login_enabled", "auth:email_registration_enabled", "auth:email_verification_enabled", "auth:email_registration_block_plus_alias", "auth:auto_link_verified_email", "chat:rag_enabled", "chat:message_embedding_enabled", "chat:semantic_context_enabled", "file:embedding_enabled", "file:embed_trigger_on_upload", "file:embedding_normalize", "extract:image_ocr_enabled", "extract:pdf_ocr_fallback_enabled", "mcp:mcp_enable":
case "auth:username_login_enabled", "auth:email_login_enabled", "auth:third_party_login_enabled", "auth:email_registration_enabled", "auth:email_verification_enabled", "auth:email_registration_block_plus_alias", "auth:auto_link_verified_email", "chat:rag_enabled", "chat:message_embedding_enabled", "chat:semantic_context_enabled", "file:full_context_limit_enabled", "file:embedding_enabled", "file:embed_trigger_on_upload", "file:embedding_normalize", "extract:image_ocr_enabled", "extract:pdf_ocr_fallback_enabled", "mcp:mcp_enable":
if _, err := strconv.ParseBool(value); err != nil {
return fmt.Errorf("%s must be bool", key)
}
Expand Down Expand Up @@ -814,6 +816,28 @@ func validateInt64Min(value string, min int64, key string) error {
return nil
}

func validateOptionalInt64Min(value string, min int64, key string) error {
if strings.TrimSpace(value) == "" {
return nil
}
v, err := strconv.ParseInt(value, 10, 64)
if err != nil || v < min {
return fmt.Errorf("%s must be empty or >= %d", key, min)
}
return nil
}

func validateOptionalIntZeroOrMinMax(value string, min int, max int, key string) error {
if strings.TrimSpace(value) == "" {
return nil
}
v, err := strconv.Atoi(value)
if err != nil || v < 0 || (v > 0 && (v < min || v > max)) {
return fmt.Errorf("%s must be empty, 0, or between %d and %d", key, min, max)
}
return nil
}

func validateFloatMinMax(value string, min float64, max float64, key string) error {
v, err := strconv.ParseFloat(strings.TrimSpace(value), 64)
if err != nil || v < min || v > max {
Expand Down
119 changes: 114 additions & 5 deletions backend/internal/application/settings/service_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,11 +135,120 @@ func TestValidateModelOptionPolicySettings(t *testing.T) {
}
}

func TestValidateFullContextMaxTokensAllowsLargeContextWindows(t *testing.T) {
if err := validatePatchItem(PatchItem{Namespace: "file", Key: "full_context_max_tokens", Value: "1000000"}); err != nil {
t.Fatalf("expected 1M full context token limit to pass, got %v", err)
func TestValidateFullContextLimitsAllowUnlimitedValues(t *testing.T) {
cases := []PatchItem{
{Namespace: "file", Key: "full_context_limit_enabled", Value: "true"},
{Namespace: "file", Key: "full_context_limit_enabled", Value: "false"},
{Namespace: "file", Key: "file_full_context_max_bytes", Value: ""},
{Namespace: "file", Key: "file_full_context_max_bytes", Value: "0"},
{Namespace: "file", Key: "full_context_max_tokens", Value: ""},
{Namespace: "file", Key: "full_context_max_tokens", Value: "0"},
{Namespace: "file", Key: "full_context_pdf_max_pages", Value: ""},
{Namespace: "file", Key: "full_context_pdf_max_pages", Value: "0"},
}

for _, item := range cases {
if err := validatePatchItem(item); err != nil {
t.Fatalf("expected %s:%s=%q to pass, got %v", item.Namespace, item.Key, item.Value, err)
}
}
}

func TestValidateFullContextLimitsEnforcesConfiguredRanges(t *testing.T) {
cases := []struct {
name string
item PatchItem
want bool
}{
{
name: "full context limit mode must be boolean",
item: PatchItem{Namespace: "file", Key: "full_context_limit_enabled", Value: "disabled"},
want: false,
},
{
name: "1M token limit passes",
item: PatchItem{Namespace: "file", Key: "full_context_max_tokens", Value: "1000000"},
want: true,
},
{
name: "token limit below minimum fails",
item: PatchItem{Namespace: "file", Key: "full_context_max_tokens", Value: "127"},
want: false,
},
{
name: "token limit above maximum fails",
item: PatchItem{Namespace: "file", Key: "full_context_max_tokens", Value: "1000001"},
want: false,
},
{
name: "negative byte limit fails",
item: PatchItem{Namespace: "file", Key: "file_full_context_max_bytes", Value: "-1"},
want: false,
},
{
name: "pdf page limit above maximum fails",
item: PatchItem{Namespace: "file", Key: "full_context_pdf_max_pages", Value: "501"},
want: false,
},
}

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
err := validatePatchItem(tc.item)
if tc.want && err != nil {
t.Fatalf("expected validation to pass, got %v", err)
}
if !tc.want && err == nil {
t.Fatal("expected validation to fail")
}
})
}
}

func TestRuntimeSettingsDisablesFullContextLimits(t *testing.T) {
runtimeSettings := NewRuntimeSettings(nil, nil, "test-data-encryption-key")
cfg := config.Config{
FileFullContextLimitEnabled: true,
FileFullContextMaxBytes: 51200,
FileFullContextMaxTokens: 12000,
FileFullContextPDFMaxPages: 20,
}
if err := validatePatchItem(PatchItem{Namespace: "file", Key: "full_context_max_tokens", Value: "1000001"}); err == nil {
t.Fatal("expected full context token limit above 1M to fail")

runtimeSettings.applyItem(&cfg, domainsettings.SystemSetting{Namespace: "file", Key: "full_context_limit_enabled", Value: "false"})
runtimeSettings.normalizeConfig(&cfg)

if cfg.FileFullContextLimitEnabled {
t.Fatal("expected full context limit switch to be disabled")
}
if cfg.FileFullContextMaxBytes != 0 || cfg.FileFullContextMaxTokens != 0 || cfg.FileFullContextPDFMaxPages != 0 {
t.Fatalf(
"expected disabled full context limits to be unlimited, got bytes=%d tokens=%d pdfPages=%d",
cfg.FileFullContextMaxBytes,
cfg.FileFullContextMaxTokens,
cfg.FileFullContextPDFMaxPages,
)
}
}

func TestRuntimeSettingsTreatsEmptyFullContextLimitsAsUnlimited(t *testing.T) {
runtimeSettings := NewRuntimeSettings(nil, nil, "test-data-encryption-key")
cfg := config.Config{
FileFullContextLimitEnabled: true,
FileFullContextMaxBytes: 51200,
FileFullContextMaxTokens: 12000,
FileFullContextPDFMaxPages: 20,
}

runtimeSettings.applyItem(&cfg, domainsettings.SystemSetting{Namespace: "file", Key: "file_full_context_max_bytes", Value: ""})
runtimeSettings.applyItem(&cfg, domainsettings.SystemSetting{Namespace: "file", Key: "full_context_max_tokens", Value: ""})
runtimeSettings.applyItem(&cfg, domainsettings.SystemSetting{Namespace: "file", Key: "full_context_pdf_max_pages", Value: ""})

if cfg.FileFullContextMaxBytes != 0 || cfg.FileFullContextMaxTokens != 0 || cfg.FileFullContextPDFMaxPages != 0 {
t.Fatalf(
"expected empty full context limits to be unlimited, got bytes=%d tokens=%d pdfPages=%d",
cfg.FileFullContextMaxBytes,
cfg.FileFullContextMaxTokens,
cfg.FileFullContextPDFMaxPages,
)
}
}
2 changes: 2 additions & 0 deletions backend/internal/infra/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,7 @@ type Config struct {
MaxMessageFiles int
// 文件处理配置
ImageMaxDimension int // 图片缩放最大边长（像素），0 = 不缩放
FileFullContextLimitEnabled bool // 是否启用全文注入阈值限制
FileFullContextMaxBytes int64 // 文本文件全文注入阈值（字节），超出不注入
FileFullContextMaxTokens int // 文本文件全文注入阈值（token）
FileImageMaxBytes int64 // 图片单文件上限（字节）
Expand Down Expand Up @@ -518,6 +519,7 @@ func Load() Config {
MaxUploadFileBytes: 20971520,
MaxMessageFiles: 10,
ImageMaxDimension: 1024,
FileFullContextLimitEnabled: true,
FileFullContextMaxBytes: 51200, // 50KB
FileFullContextMaxTokens: 12000,
FileImageMaxBytes: 0,
Expand Down
49 changes: 46 additions & 3 deletions frontend/features/admin/model/chat-files.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,11 @@ export const EMBEDDING_MODES = {
ON: "true",
} as const;

export const FULL_CONTEXT_LIMIT_MODES = {
OFF: "false",
ON: "true",
} as const;

export const MINERU_SERVICE_SOURCES = {
CLOUD: "cloud",
SELF_HOSTED: "self_hosted",
Expand Down Expand Up @@ -593,9 +598,44 @@ export const SETTINGS_GROUPS: SettingsGroup[] = [
title: "Full-text injection",
description: "Controls whether extracted file text can be injected directly into context instead of using RAG.",
fields: [
{ namespace: "file", key: "file_full_context_max_bytes", label: "Full-text byte limit", description: "Maximum extracted text bytes allowed for full-context injection.", type: "int", placeholder: "Byte limit" },
{ namespace: "file", key: "full_context_max_tokens", label: "Full-text token limit", description: "Maximum token budget allowed for full-context injection.", type: "int", placeholder: "Token limit" },
{ namespace: "file", key: "full_context_pdf_max_pages", label: "Full-text page limit", description: "Maximum PDF pages allowed for full-context injection.", type: "int", placeholder: "Max pages" },
{
namespace: "file",
key: "full_context_limit_enabled",
label: "Full-text injection limits",
description: "Enable byte, token, and PDF page limits for full-text injection.",
type: "select",
options: [
{ label: "Off", value: FULL_CONTEXT_LIMIT_MODES.OFF },
{ label: "On", value: FULL_CONTEXT_LIMIT_MODES.ON },
],
},
{
namespace: "file",
key: "file_full_context_max_bytes",
label: "Full-text byte limit",
description: "Maximum extracted text bytes allowed for full-context injection. Leave empty or use 0 for no byte limit.",
type: "int",
placeholder: "Empty or 0 means unlimited",
visibleWhen: { field: "file.full_context_limit_enabled", equals: FULL_CONTEXT_LIMIT_MODES.ON },
},
{
namespace: "file",
key: "full_context_max_tokens",
label: "Full-text token limit",
description: "Maximum token budget allowed for full-context injection. Leave empty or use 0 for no token limit.",
type: "int",
placeholder: "Empty or 0 means unlimited",
visibleWhen: { field: "file.full_context_limit_enabled", equals: FULL_CONTEXT_LIMIT_MODES.ON },
},
{
namespace: "file",
key: "full_context_pdf_max_pages",
label: "Full-text page limit",
description: "Maximum PDF pages allowed for full-context injection. Leave empty or use 0 for no page limit.",
type: "int",
placeholder: "Empty or 0 means unlimited",
visibleWhen: { field: "file.full_context_limit_enabled", equals: FULL_CONTEXT_LIMIT_MODES.ON },
},
],
},
{
Expand Down Expand Up @@ -1065,6 +1105,9 @@ export function applySettingsDefaults(next: Record<string, string>): Record<stri
if (!["true", "false"].includes(result["file.embedding_enabled"] ?? "")) {
result["file.embedding_enabled"] = EMBEDDING_MODES.OFF;
}
if (!["true", "false"].includes(result["file.full_context_limit_enabled"] ?? "")) {
result["file.full_context_limit_enabled"] = FULL_CONTEXT_LIMIT_MODES.ON;
}
if (!["true", "false"].includes(result["chat.rag_enabled"] ?? "")) {
result["chat.rag_enabled"] = "false";
}
Expand Down
20 changes: 14 additions & 6 deletions frontend/i18n/messages/en-US/admin-files.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,20 +102,28 @@
"description": "Override the size limit for document attachments. Falls back to the default attachment limit when empty.",
"placeholder": "Leave empty to use the default attachment limit"
},
"full_context_limit_enabled": {
"label": "Full-text injection limits",
"description": "When enabled, full-text injection is limited by byte size, token budget, and PDF pages. When disabled, these limits are ignored.",
"options": {
"false": "Off",
"true": "On"
}
},
"file_full_context_max_bytes": {
"label": "Full-text byte limit",
"description": "Maximum extracted text bytes allowed for full-context injection.",
"placeholder": "Byte limit"
"description": "Maximum extracted text bytes allowed for full-context injection. Leave empty or use 0 for no byte limit.",
"placeholder": "Empty or 0 means unlimited"
},
"full_context_max_tokens": {
"label": "Full-text token limit",
"description": "Maximum token budget allowed for full-context injection.",
"placeholder": "Token limit"
"description": "Maximum token budget allowed for full-context injection. Leave empty or use 0 for no token limit.",
"placeholder": "Empty or 0 means unlimited"
},
"full_context_pdf_max_pages": {
"label": "Full-text page limit",
"description": "Maximum PDF pages allowed for full-context injection.",
"placeholder": "Max pages"
"description": "Maximum PDF pages allowed for full-context injection. Leave empty or use 0 for no page limit.",
"placeholder": "Empty or 0 means unlimited"
},
"embedding_enabled": {
"label": "Embedding",
Expand Down
Loading
Loading