feat(api): /v1/answer/pageindex endpoint with reasoning + streaming

Wire the PageIndex strategy through a dedicated answer endpoint
on the existing /v1 router. The endpoint:

  - Owns the full RAG round-trip in one request: retrieval +
    answer + citations come back from a single agentic loop.
    No separate synthesis call — the model emits its answer
    inside the done action and we surface it as `answer` on
    the response.

  - Emits page-grounded citations. One citation per page range
    the agent fetched (deduplicated), each carrying
    start_page / end_page / section_ids plus an answer-span
    quote pulled via the existing SpanExtractor over the cited
    content. Falls back gracefully when the LLM declines a
    quote.

  - Persists every successful response to the existing replay
    store under the strategy's deterministic trace_token. The
    token's input set is sorted cited page ranges (not section
    IDs), and the strategy name is folded into the hash so
    page-based and section-based tokens for the same doc/model
    never collide.

  - Supports an opt-in reasoning trace (body field
    `reasoning:true` or query param `?reasoning=true`) that
    surfaces per-hop tool calls + args + tool-result chars +
    sections touched, captured via a new OnEvent hook on
    PageIndexStrategy.

  - Streams via Server-Sent Events when `stream:true` is set
    on the body. One event per tool call (get_document_structure,
    get_pages, done) so callers WATCH the navigation in real
    time, terminated by an `answer` event carrying the full
    JSON response payload.

  - Honors per-request overrides for max_hops and
    max_pages_per_fetch without mutating shared Deps. Disabled
    deployments (retrieval.pageindex.enabled=false or no LLM
    client) return 501; missing documents 404; bad bodies 400.

Adds `RetrievalConfig.PageIndex` (PageIndexBlock) with defaults
(Enabled=true, MaxHops=8, PageContentLimit=16000) and matching
VLE_RETRIEVAL_PAGEINDEX_* env overrides. Validation rejects
negative knobs and accepts "pageindex" as a retrieval strategy.

cmd/engine/main.go registers the strategy via buildStrategy
when retrieval.strategy=pageindex, AND wires a standalone
PageIndexStrategy instance into the api.Deps used by the
answer endpoint — so the endpoint is available regardless of
which selection strategy the deployment runs by default.

Test coverage: 12 end-to-end handler tests (happy path,
reasoning trace via body field + query param, bad request,
not found, disabled in two modes, no LLM, replay persistence
verifying byte-equal response bytes, SSE event stream shape,
per-request override caps the loop, TOC fallback). Plus 5
config tests for defaults + env overrides + validation.

A PageIndexTreeLoader function field on Deps acts as a test
seam so handler tests can run end-to-end via httptest with
an in-memory tree, without a real Postgres backend.

Author: hallelx2

cmd/engine/main.go

@@ -196,24 +196,41 @@ func run() error {
 	}
 	q.Register(queue.KindIngestDocument, pipeline.Handler())
 
+	// /v1/answer/pageindex gets its OWN PageIndexStrategy instance,
+	// independent of whatever selection strategy is configured in
+	// retrieval.strategy. This way the endpoint is always available
+	// (gated by retrieval.pageindex.enabled), even on a deployment
+	// using chunked-tree as its default selection path.
+	var pageIndexStrategy *retrieval.PageIndexStrategy
+	if cfg.Retrieval.PageIndex.Enabled && llmClient != nil {
+		pageIndexStrategy = buildPageIndexStrategy(cfg.Retrieval, llmClient, store)
+		logger.Info("retrieval: pageindex answer endpoint enabled",
+			"max_hops", pageIndexStrategy.MaxHops,
+			"page_content_limit", pageIndexStrategy.PageContentLimit,
+			"model_override", cfg.Retrieval.PageIndex.Model,
+		)
+	}
+
 	deps := api.Deps{
-		Logger:     logger,
-		DB:         pool,
-		Storage:    store,
-		Queue:      q,
-		Strategy:   strategy,
-		Version:    version,
-		MultiDoc:   multiDoc,
-		LLM:        llmClient,
-		LLMModel:   modelFor(cfg.LLM),
-		AnswerSpan: cfg.Retrieval.AnswerSpan,
-		Answer:     cfg.Retrieval.Answer,
-		Planner:    planner,
-		Planning:   cfg.Retrieval.Planning,
-		ReRanker:   reRanker,
-		ReRank:     cfg.Retrieval.ReRank,
-		Replay:     replayStore,
-		Abstain:    cfg.Retrieval.Abstain,
+		Logger:            logger,
+		DB:                pool,
+		Storage:           store,
+		Queue:             q,
+		Strategy:          strategy,
+		Version:           version,
+		MultiDoc:          multiDoc,
+		LLM:               llmClient,
+		LLMModel:          modelFor(cfg.LLM),
+		AnswerSpan:        cfg.Retrieval.AnswerSpan,
+		Answer:            cfg.Retrieval.Answer,
+		Planner:           planner,
+		Planning:          cfg.Retrieval.Planning,
+		ReRanker:          reRanker,
+		ReRank:            cfg.Retrieval.ReRank,
+		Replay:            replayStore,
+		Abstain:           cfg.Retrieval.Abstain,
+		PageIndexStrategy: pageIndexStrategy,
+		PageIndex:         cfg.Retrieval.PageIndex,
 	}
 
 	srv := &http.Server{
@@ -365,11 +382,36 @@ func buildStrategy(c config.RetrievalConfig, client llmgate.Client, store storag
 		}
 		a.ModelOverride = c.Agentic.Model
 		return a
+	case "pageindex":
+		return buildPageIndexStrategy(c, client, store)
 	default:
 		return retrieval.NewChunkedTree(client)
 	}
 }
 
+// buildPageIndexStrategy constructs the page-based agentic
+// strategy with the storage-backed PageLoader and the configured
+// caps. Used by buildStrategy when retrieval.strategy=pageindex AND
+// by the /v1/answer/pageindex endpoint setup (which wires its own
+// instance regardless of the selection strategy).
+//
+// The TOCProvider is left nil here. PR-A (toc-tree-builder) adds
+// documents.toc_tree + a DB-backed provider; until it lands the
+// strategy degrades to its synthesised view, which is the
+// documented fallback path.
+func buildPageIndexStrategy(c config.RetrievalConfig, client llmgate.Client, store storage.Storage) *retrieval.PageIndexStrategy {
+	p := retrieval.NewPageIndexStrategy(client)
+	p.PageLoader = storagePageLoader{s: store}
+	if c.PageIndex.MaxHops > 0 {
+		p.MaxHops = c.PageIndex.MaxHops
+	}
+	if c.PageIndex.PageContentLimit > 0 {
+		p.PageContentLimit = c.PageIndex.PageContentLimit
+	}
+	p.ModelOverride = c.PageIndex.Model
+	return p
+}
+
 // storageFetcher adapts a storage.Storage to retrieval.ContentFetcher.
 // The agentic strategy reads section bodies one at a time, so we
 // materialize the full reader contents into a []byte here rather than
@@ -385,6 +427,23 @@ func (sf storageFetcher) Get(ctx context.Context, ref string) ([]byte, error) {
 	return io.ReadAll(rc)
 }
 
+// storagePageLoader adapts a storage.Storage to
+// retrieval.PageContentLoader. Mirrors storageFetcher but lives
+// behind a separate interface so the two callers (agentic /
+// pageindex) can be wired independently. The PageIndex strategy
+// materialises section bodies once per get_pages observation, so
+// reading the full reader into a []byte is the right shape.
+type storagePageLoader struct{ s storage.Storage }
+
+func (l storagePageLoader) Load(ctx context.Context, ref string) ([]byte, error) {
+	rc, _, err := l.s.Get(ctx, ref)
+	if err != nil {
+		return nil, err
+	}
+	defer rc.Close()
+	return io.ReadAll(rc)
+}
+
 // buildTLSConfig returns a *tls.Config when direct TLS is enabled, or nil
 // when the engine should serve plaintext (behind a proxy). Returning nil
 // leaves http.Server's TLSConfig unset, which is exactly what ListenAndServe