Skip to content

Commit 0f9a2cf

Browse files
committed
test(ingest): prove minimal mode does zero LLM work and stays queryable
- pkg/ingest/minimal_mode_test.go: a minimal-mode pipeline run with an LLM client that fails the test on any call reaches StatusReady with sections persisted and a call counter of 0 — proving minimal ingest is pure-Go. A second test reconstructs the persisted tree and confirms the synthesised-TOC fallback is title-bearing and section bodies load back from storage. - pkg/retrieval: TestPageIndexMinimalIngestedDoc drives the page-based strategy end-to-end against a minimal-ingested doc shape (page ranges + content refs, NO summaries, nil TOC) and asserts it produces a cited answer from the synthesised TOC + raw page reads. - pkg/config: default mode is "full"; VLE_INGEST_MODE=minimal override and Validate accept/reject coverage. - Document ingest.mode in both example configs.
1 parent 6444532 commit 0f9a2cf

5 files changed

Lines changed: 483 additions & 0 deletions

File tree

config.example.yaml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -294,10 +294,37 @@ retrieval:
294294
model: ""
295295

296296
ingest:
297+
# Ingest mode — how much work the pipeline does before a document is
298+
# marked `ready` (queryable).
299+
#
300+
# full (default) parse -> build tree -> persist -> summarize ->
301+
# HyDE -> multi-axis summaries -> TOC build. Maximises
302+
# retrieval quality but costs ~1,000-3,000 LLM calls plus a
303+
# pdftable table-finding pass on a large filing — minutes of
304+
# wall time for a 90-page 10-K.
305+
#
306+
# minimal parse -> build tree -> persist -> ready. Skips ALL
307+
# per-section LLM enrichment (summarize, HyDE, multi-axis,
308+
# TOC build) AND the pdftable table-extraction pass, so a
309+
# document becomes queryable in ~parse-speed (seconds).
310+
# The page-based strategy (/v1/answer/pageindex) needs none
311+
# of the skipped work: it navigates a TOC synthesised from
312+
# the section tree (documents.toc_tree is left NULL) and
313+
# reads raw section/page text at query time — and that raw
314+
# page text still contains the tables' text, so dropping
315+
# table *sections* loses nothing for it. The
316+
# summary-dependent strategies (chunked-tree, agentic)
317+
# degrade to titles + raw content with no summaries.
318+
#
319+
# Override per-process with VLE_INGEST_MODE; on the deployed
320+
# vectorless-server use VLS_INGEST_MODE=minimal (no secret edit needed).
321+
mode: "full"
322+
297323
# The summarize and HyDE stages run concurrently. This caps the total
298324
# number of LLM calls in flight across both stages combined, so the
299325
# provider's per-tenant concurrency limit isn't exceeded. 0 disables
300326
# the global cap; default applied by the engine is 12.
327+
# (Ignored when mode: minimal — no LLM stages run.)
301328
global_llm_concurrency: 12
302329

303330
# HyDE candidate-question stage. For each leaf section the pipeline asks

config.server.example.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,9 +99,21 @@ engine:
9999
include_sibling_breadcrumbs: true
100100

101101
ingest:
102+
# Ingest mode: full (default) | minimal.
103+
# full parse -> persist -> summarize -> HyDE -> multi-axis ->
104+
# TOC build. Maximum retrieval quality; minutes on a large
105+
# filing.
106+
# minimal parse -> persist -> ready. Skips every LLM enrichment
107+
# stage AND table extraction — queryable in seconds. The
108+
# page-based strategy (/v1/answer/pageindex) works on it
109+
# unchanged (synthesised TOC + raw page reads).
110+
# Flip the live service without a secret edit: VLS_INGEST_MODE=minimal.
111+
mode: "full"
112+
102113
# The summarize and HyDE stages run concurrently. This caps the total
103114
# number of LLM calls in flight across both stages combined.
104115
# 0 disables the global cap; default is 12.
116+
# (Ignored when mode: minimal — no LLM stages run.)
105117
global_llm_concurrency: 12
106118

107119
# HyDE candidate-question generation per leaf section. Folded into

pkg/config/config_test.go

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,52 @@ func TestDefaultValues(t *testing.T) {
8484
}
8585
}
8686

87+
// TestIngestModeDefault locks the default ingest mode to "full" so the
88+
// current full-enrichment behaviour is preserved unless explicitly
89+
// switched.
90+
func TestIngestModeDefault(t *testing.T) {
91+
t.Parallel()
92+
cfg := Default()
93+
if cfg.Ingest.Mode != "full" {
94+
t.Errorf("ingest.mode = %q, want full (default)", cfg.Ingest.Mode)
95+
}
96+
}
97+
98+
// TestIngestModeEnvOverride covers the VLE_INGEST_MODE override — the
99+
// single env var that flips the engine into fast/minimal ingest.
100+
func TestIngestModeEnvOverride(t *testing.T) {
101+
prev := os.Getenv("VLE_INGEST_MODE")
102+
defer os.Setenv("VLE_INGEST_MODE", prev)
103+
104+
os.Setenv("VLE_INGEST_MODE", "minimal")
105+
cfg := Default()
106+
applyEnvOverrides(&cfg)
107+
if cfg.Ingest.Mode != "minimal" {
108+
t.Errorf("VLE_INGEST_MODE=minimal not applied, got %q", cfg.Ingest.Mode)
109+
}
110+
}
111+
112+
// TestIngestModeValidate asserts Validate accepts the documented values
113+
// (and empty, which Default normalises to full) and rejects garbage.
114+
func TestIngestModeValidate(t *testing.T) {
115+
t.Parallel()
116+
for _, m := range []string{"", "full", "minimal"} {
117+
cfg := Default()
118+
cfg.Database.URL = "postgres://localhost/test"
119+
cfg.Ingest.Mode = m
120+
if err := cfg.Validate(); err != nil {
121+
t.Errorf("ingest.mode=%q should pass validation, got %v", m, err)
122+
}
123+
}
124+
125+
cfg := Default()
126+
cfg.Database.URL = "postgres://localhost/test"
127+
cfg.Ingest.Mode = "turbo"
128+
if err := cfg.Validate(); err == nil {
129+
t.Error("ingest.mode=turbo should fail validation")
130+
}
131+
}
132+
87133
func TestTOCEnvOverride(t *testing.T) {
88134
// Mutates env — restore on exit. Not parallel.
89135
keys := []string{

0 commit comments

Comments
 (0)