Merge branch 'main' into feat/agentic-retrieval

Author: hallelx2

cmd/engine/main.go

@@ -110,11 +110,15 @@ func run() error {
 	multiDoc := retrieval.NewMultiDoc(strategy, pool.LoadTree)
 
 	pipeline := ingest.NewPipeline(ingest.Pipeline{
-		DB:      pool,
-		Storage: store,
-		LLM:     llmClient,
-		Parsers: ingest.DefaultRegistry(),
-		Logger:  logger,
+		DB:               pool,
+		Storage:          store,
+		LLM:              llmClient,
+		Parsers:          ingest.DefaultRegistry(),
+		Logger:           logger,
+		HyDEEnabled:      cfg.Ingest.HyDE.Enabled,
+		HyDEModel:        cfg.Ingest.HyDE.Model,
+		HyDENumQuestions: cfg.Ingest.HyDE.NumQuestions,
+		HyDEConcurrency:  cfg.Ingest.HyDE.Concurrency,
 	})
 	q.Register(queue.KindIngestDocument, pipeline.Handler())
 

cmd/server/main.go

@@ -155,11 +155,15 @@ func run() error {
 
 	// ── Ingest pipeline ───────────────────────────────────────────
 	pipeline := ingest.NewPipeline(ingest.Pipeline{
-		DB:      pool,
-		Storage: store,
-		LLM:     llmClient,
-		Parsers: ingest.DefaultRegistry(),
-		Logger:  logger,
+		DB:               pool,
+		Storage:          store,
+		LLM:              llmClient,
+		Parsers:          ingest.DefaultRegistry(),
+		Logger:           logger,
+		HyDEEnabled:      cfg.Engine.Ingest.HyDE.Enabled,
+		HyDEModel:        cfg.Engine.Ingest.HyDE.Model,
+		HyDENumQuestions: cfg.Engine.Ingest.HyDE.NumQuestions,
+		HyDEConcurrency:  cfg.Engine.Ingest.HyDE.Concurrency,
 	})
 	q.Register(queue.KindIngestDocument, pipeline.Handler())
 

config.example.yaml

@@ -107,6 +107,18 @@ retrieval:
     # doesn't own, so the model knows what else exists in the document.
     include_sibling_breadcrumbs: true
 
+ingest:
+  # HyDE candidate-question stage. For each leaf section the pipeline asks
+  # the LLM to enumerate questions the section answers; those are folded
+  # into the retrieval prompt at query time to widen recall on queries
+  # that don't echo the section's exact wording.
+  hyde:
+    enabled: true
+    # Override the LLM model used for HyDE; empty inherits the summary model.
+    model: ""
+    num_questions: 5
+    concurrency: 4
+
 log:
   level: "info"            # debug | info | warn | error
   format: "json"           # json | console

config.server.example.yaml

@@ -98,6 +98,16 @@ engine:
       max_parallel_calls: 8
       include_sibling_breadcrumbs: true
 
+  ingest:
+    # HyDE candidate-question generation per leaf section. Folded into
+    # the retrieval prompt at query time to widen recall on queries that
+    # don't echo the section's exact wording.
+    hyde:
+      enabled: true
+      model: ""             # empty => same model as summarization
+      num_questions: 5
+      concurrency: 4
+
   log:
     level: "info"     # "debug", "info", "warn", "error"
     format: "json"    # "json" or "console"

internal/api/server.go

@@ -321,7 +321,7 @@ func (d Deps) handleGetSection(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	writeJSON(w, http.StatusOK, map[string]any{
+	resp := map[string]any{
 		"id":          sec.ID,
 		"document_id": sec.DocumentID,
 		"parent_id":   sec.ParentID,
@@ -332,7 +332,17 @@ func (d Deps) handleGetSection(w http.ResponseWriter, r *http.Request) {
 		"token_count": sec.TokenCount,
 		"metadata":    sec.Metadata,
 		"content":     content,
-	})
+	}
+	if sec.PageStart > 0 {
+		resp["page_start"] = sec.PageStart
+	}
+	if sec.PageEnd > 0 {
+		resp["page_end"] = sec.PageEnd
+	}
+	if len(sec.CandidateQuestions) > 0 {
+		resp["candidate_questions"] = sec.CandidateQuestions
+	}
+	writeJSON(w, http.StatusOK, resp)
 }
 
 // --- query ---
@@ -415,14 +425,24 @@ func (d Deps) handleQuery(w http.ResponseWriter, r *http.Request) {
 				content = string(raw)
 			}
 		}
-		sections = append(sections, map[string]any{
+		s := map[string]any{
 			"id":          sec.ID,
 			"parent_id":   sec.ParentID,
 			"title":       sec.Title,
 			"summary":     sec.Summary,
 			"token_count": sec.TokenCount,
 			"content":     content,
-		})
+		}
+		if sec.PageStart > 0 {
+			s["page_start"] = sec.PageStart
+		}
+		if sec.PageEnd > 0 {
+			s["page_end"] = sec.PageEnd
+		}
+		if len(sec.CandidateQuestions) > 0 {
+			s["candidate_questions"] = sec.CandidateQuestions
+		}
+		sections = append(sections, s)
 	}
 
 	writeJSON(w, http.StatusOK, map[string]any{
@@ -512,6 +532,15 @@ func (d Deps) handleQueryMulti(w http.ResponseWriter, r *http.Request) {
 				"token_count": sec.TokenCount,
 				"content":     content,
 			}
+			if sec.PageStart > 0 {
+				s["page_start"] = sec.PageStart
+			}
+			if sec.PageEnd > 0 {
+				s["page_end"] = sec.PageEnd
+			}
+			if len(sec.CandidateQuestions) > 0 {
+				s["candidate_questions"] = sec.CandidateQuestions
+			}
 			sections = append(sections, s)
 			if body.MaxSections > 0 && len(sections) >= body.MaxSections {
 				break

openapi.yaml

@@ -375,6 +375,17 @@ components:
           type: string
         token_count:
           type: integer
+        page_start:
+          type: integer
+          description: Inclusive first page covered by this section. Omitted for non-paginated formats.
+        page_end:
+          type: integer
+          description: Inclusive last page covered by this section. Omitted for non-paginated formats.
+        candidate_questions:
+          type: array
+          items:
+            type: string
+          description: HyDE-generated questions this section can answer. Omitted when not yet generated.
         metadata:
           type: object
           additionalProperties:
@@ -440,6 +451,17 @@ components:
           type: string
         token_count:
           type: integer
+        page_start:
+          type: integer
+          description: Inclusive first page covered by this section. Omitted for non-paginated formats.
+        page_end:
+          type: integer
+          description: Inclusive last page covered by this section. Omitted for non-paginated formats.
+        candidate_questions:
+          type: array
+          items:
+            type: string
+          description: HyDE-generated questions this section can answer. Omitted when not yet generated.
         content:
           type: string
           description: Full section content from storage.

pkg/config/config.go

@@ -12,6 +12,7 @@ import (
 	"fmt"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"gopkg.in/yaml.v3"
@@ -25,9 +26,38 @@ type Config struct {
 	Queue     QueueConfig     `yaml:"queue"`
 	LLM       LLMConfig       `yaml:"llm"`
 	Retrieval RetrievalConfig `yaml:"retrieval"`
+	Ingest    IngestConfig    `yaml:"ingest"`
 	Log       LogConfig       `yaml:"log"`
 }
 
+// IngestConfig configures retrieval-quality boosters that run during
+// the ingest pipeline (between summarize and StatusReady).
+type IngestConfig struct {
+	HyDE HyDEConfig `yaml:"hyde"`
+}
+
+// HyDEConfig configures the HyDE candidate-question stage. For each
+// leaf section the pipeline asks the LLM to enumerate questions the
+// section's content can answer; those are later folded into the
+// retrieval prompt to widen lexical/semantic overlap with user queries.
+type HyDEConfig struct {
+	// Enabled toggles the stage. Default: true. Disable to skip an LLM
+	// call per leaf when ingest budget matters more than recall.
+	Enabled bool `yaml:"enabled"`
+
+	// Model, when non-empty, overrides the LLM model used for HyDE
+	// generation. Defaults to the same model used for summarization.
+	Model string `yaml:"model"`
+
+	// NumQuestions caps the questions generated per leaf section.
+	// Default: 5.
+	NumQuestions int `yaml:"num_questions"`
+
+	// Concurrency bounds parallel LLM calls during the HyDE stage.
+	// Default: 4.
+	Concurrency int `yaml:"concurrency"`
+}
+
 // ServerConfig configures the HTTP server.
 //
 // TLS is opt-in. If TLS.CertFile and TLS.KeyFile are both set the engine
@@ -241,6 +271,13 @@ func Default() Config {
 				TTLSeconds: 600,
 			},
 		},
+		Ingest: IngestConfig{
+			HyDE: HyDEConfig{
+				Enabled:      true,
+				NumQuestions: 5,
+				Concurrency:  4,
+			},
+		},
 		Log: LogConfig{Level: "info", Format: "json"},
 	}
 }
@@ -344,6 +381,29 @@ func applyEnvOverrides(c *Config) {
 	if v := os.Getenv("VLE_RETRIEVAL_AGENTIC_MODEL"); v != "" {
 		c.Retrieval.Agentic.Model = v
 	}
+	// Ingest / HyDE knobs. Booleans accept the usual truthy strings —
+	// kept narrow so a typo doesn't silently flip the flag.
+	if v := os.Getenv("VLE_INGEST_HYDE_ENABLED"); v != "" {
+		switch strings.ToLower(strings.TrimSpace(v)) {
+		case "1", "true", "yes", "on":
+			c.Ingest.HyDE.Enabled = true
+		case "0", "false", "no", "off":
+			c.Ingest.HyDE.Enabled = false
+		}
+	}
+	if v := os.Getenv("VLE_INGEST_HYDE_MODEL"); v != "" {
+		c.Ingest.HyDE.Model = v
+	}
+	if v := os.Getenv("VLE_INGEST_HYDE_NUM_QUESTIONS"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			c.Ingest.HyDE.NumQuestions = n
+		}
+	}
+	if v := os.Getenv("VLE_INGEST_HYDE_CONCURRENCY"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			c.Ingest.HyDE.Concurrency = n
+		}
+	}
 }
 
 // Validate checks that required fields for the selected drivers are set.
@@ -416,5 +476,12 @@ func (c Config) Validate() error {
 		return fmt.Errorf("server.tls.min_version must be 1.2 or 1.3, got %q", v)
 	}
 
+	if c.Ingest.HyDE.NumQuestions < 0 {
+		return fmt.Errorf("ingest.hyde.num_questions must be >= 0, got %d", c.Ingest.HyDE.NumQuestions)
+	}
+	if c.Ingest.HyDE.Concurrency < 0 {
+		return fmt.Errorf("ingest.hyde.concurrency must be >= 0, got %d", c.Ingest.HyDE.Concurrency)
+	}
+
 	return nil
 }

pkg/db/migrations/0004_sections_extras.down.sql

@@ -0,0 +1,5 @@
+DROP INDEX IF EXISTS sections_doc_pages_idx;
+ALTER TABLE sections
+    DROP COLUMN IF EXISTS candidate_questions,
+    DROP COLUMN IF EXISTS page_end,
+    DROP COLUMN IF EXISTS page_start;

pkg/db/migrations/0004_sections_extras.up.sql

@@ -0,0 +1,22 @@
+-- 0004_sections_extras.up.sql — page citations + HyDE candidate questions.
+--
+-- Two retrieval-quality extensions to the sections table:
+--
+--  page_start / page_end
+--      The inclusive page range each section covers, for parsers that
+--      produce page-aware output (PDF today; others leave them NULL/0).
+--      Surfaced to API responses so callers can render citations.
+--
+--  candidate_questions
+--      JSONB array of generated questions a section can answer (HyDE).
+--      Filled by the ingest pipeline's HyDE stage and woven into the
+--      retrieval prompt to widen lexical/semantic overlap with the user
+--      query.
+
+ALTER TABLE sections
+    ADD COLUMN IF NOT EXISTS page_start          INTEGER,
+    ADD COLUMN IF NOT EXISTS page_end            INTEGER,
+    ADD COLUMN IF NOT EXISTS candidate_questions JSONB;
+
+CREATE INDEX IF NOT EXISTS sections_doc_pages_idx
+    ON sections (document_id, page_start, page_end);