@@ -35,6 +35,12 @@ type Config struct {
3535type IngestConfig struct {
3636 HyDE HyDEConfig `yaml:"hyde"`
3737
38+ // Tables configures pdftable's table-finding pass over PDF inputs.
39+ // Enabled by default — tables are the single biggest retrieval-quality
40+ // boost on FinanceBench-style documents because every numeric question
41+ // hides in a balance sheet that text-only extraction collapses.
42+ Tables TablesConfig `yaml:"tables"`
43+
3844 // GlobalLLMConcurrency caps the total number of LLM calls in flight
3945 // across the summarize and HyDE stages combined, which now run
4046 // concurrently. Each stage still respects its own per-stage cap
@@ -47,6 +53,48 @@ type IngestConfig struct {
4753 GlobalLLMConcurrency int `yaml:"global_llm_concurrency"`
4854}
4955
56+ // TablesConfig configures the table-extraction stage of the PDF parser.
57+ // The stage runs pdftable's geometry-based finder over every page and
58+ // emits each detected table as its own Section with
59+ // Metadata["table"]="true", so downstream retrieval and the agentic
60+ // navigator can branch on whether a candidate is a numeric table or
61+ // prose.
62+ //
63+ // All knobs are forwarded to pdftable's TableSettings; defaults match
64+ // pdfplumber. See pdftable's docs for the full strategy surface.
65+ type TablesConfig struct {
66+ // Enabled toggles the stage. Default: true. Flip to false to
67+ // restore pre-integration text-only output; one config change is
68+ // enough to roll back if a real-world PDF triggers a regression.
69+ Enabled bool `yaml:"enabled"`
70+
71+ // VerticalStrategy picks the source of vertical column boundaries.
72+ // Allowed values:
73+ // - "lines" (default) edges from drawn lines/rects/curves
74+ // - "lines_strict" edges from drawn lines only
75+ // - "text" edges inferred from word alignment (borderless
76+ // tables — bank statements, narrative 10-Ks)
77+ // - "explicit" caller-supplied coordinates (not yet wired
78+ // through the engine config; reserved)
79+ VerticalStrategy string `yaml:"vertical_strategy"`
80+
81+ // HorizontalStrategy picks the source of horizontal row boundaries.
82+ // Same value set as VerticalStrategy; the two axes can mix
83+ // independently (e.g. "lines" vertical + "text" horizontal).
84+ HorizontalStrategy string `yaml:"horizontal_strategy"`
85+
86+ // MinTableRows drops candidate tables with fewer than this many
87+ // rows. Default: 2. Trivial single-row matches are almost always
88+ // false positives from layout artefacts (form-field grids, ruling
89+ // hairlines on a single line of text).
90+ MinTableRows int `yaml:"min_table_rows"`
91+
92+ // MinTableCols drops candidate tables with fewer than this many
93+ // columns. Default: 2. Same rationale as MinTableRows — a single
94+ // column is a vertical list, not a table.
95+ MinTableCols int `yaml:"min_table_cols"`
96+ }
97+
5098// HyDEConfig configures the HyDE candidate-question stage. For each
5199// leaf section the pipeline asks the LLM to enumerate questions the
52100// section's content can answer; those are later folded into the
@@ -449,6 +497,13 @@ func Default() Config {
449497 NumQuestions : 5 ,
450498 Concurrency : 4 ,
451499 },
500+ Tables : TablesConfig {
501+ Enabled : true ,
502+ VerticalStrategy : "lines" ,
503+ HorizontalStrategy : "lines" ,
504+ MinTableRows : 2 ,
505+ MinTableCols : 2 ,
506+ },
452507 },
453508 Log : LogConfig {Level : "info" , Format : "json" },
454509 }
@@ -581,6 +636,31 @@ func applyEnvOverrides(c *Config) {
581636 c .Ingest .GlobalLLMConcurrency = n
582637 }
583638 }
639+ // pdftable-driven table extraction.
640+ if v := os .Getenv ("VLE_INGEST_TABLES_ENABLED" ); v != "" {
641+ switch strings .ToLower (strings .TrimSpace (v )) {
642+ case "1" , "true" , "yes" , "on" :
643+ c .Ingest .Tables .Enabled = true
644+ case "0" , "false" , "no" , "off" :
645+ c .Ingest .Tables .Enabled = false
646+ }
647+ }
648+ if v := os .Getenv ("VLE_INGEST_TABLES_VERTICAL_STRATEGY" ); v != "" {
649+ c .Ingest .Tables .VerticalStrategy = v
650+ }
651+ if v := os .Getenv ("VLE_INGEST_TABLES_HORIZONTAL_STRATEGY" ); v != "" {
652+ c .Ingest .Tables .HorizontalStrategy = v
653+ }
654+ if v := os .Getenv ("VLE_INGEST_TABLES_MIN_ROWS" ); v != "" {
655+ if n , err := strconv .Atoi (v ); err == nil && n >= 0 {
656+ c .Ingest .Tables .MinTableRows = n
657+ }
658+ }
659+ if v := os .Getenv ("VLE_INGEST_TABLES_MIN_COLS" ); v != "" {
660+ if n , err := strconv .Atoi (v ); err == nil && n >= 0 {
661+ c .Ingest .Tables .MinTableCols = n
662+ }
663+ }
584664 if v := os .Getenv ("VLE_RETRIEVAL_ANSWER_SPAN_ENABLED" ); v != "" {
585665 switch strings .ToLower (strings .TrimSpace (v )) {
586666 case "1" , "true" , "yes" , "on" :
@@ -750,6 +830,25 @@ func (c Config) Validate() error {
750830 return fmt .Errorf ("ingest.global_llm_concurrency must be >= 0, got %d" , c .Ingest .GlobalLLMConcurrency )
751831 }
752832
833+ switch c .Ingest .Tables .VerticalStrategy {
834+ case "" , "lines" , "lines_strict" , "text" , "explicit" :
835+ default :
836+ return fmt .Errorf ("ingest.tables.vertical_strategy must be one of lines|lines_strict|text|explicit, got %q" ,
837+ c .Ingest .Tables .VerticalStrategy )
838+ }
839+ switch c .Ingest .Tables .HorizontalStrategy {
840+ case "" , "lines" , "lines_strict" , "text" , "explicit" :
841+ default :
842+ return fmt .Errorf ("ingest.tables.horizontal_strategy must be one of lines|lines_strict|text|explicit, got %q" ,
843+ c .Ingest .Tables .HorizontalStrategy )
844+ }
845+ if c .Ingest .Tables .MinTableRows < 0 {
846+ return fmt .Errorf ("ingest.tables.min_table_rows must be >= 0, got %d" , c .Ingest .Tables .MinTableRows )
847+ }
848+ if c .Ingest .Tables .MinTableCols < 0 {
849+ return fmt .Errorf ("ingest.tables.min_table_cols must be >= 0, got %d" , c .Ingest .Tables .MinTableCols )
850+ }
851+
753852 if c .Retrieval .Planning .CacheSize < 0 {
754853 return fmt .Errorf ("retrieval.planning.cache_size must be >= 0, got %d" , c .Retrieval .Planning .CacheSize )
755854 }
0 commit comments