@@ -134,6 +134,29 @@ type Pipeline struct {
134134 // per-stage semaphore). Default applied by NewPipeline: 12.
135135 GlobalLLMConcurrency int
136136
137+ // TOCEnabled toggles the LLM-built table-of-contents stage. The
138+ // stage runs after summarize+HyDE on PDF inputs and persists the
139+ // resulting tree on documents.toc_tree (JSONB). Failures are
140+ // non-fatal — they leave the column NULL.
141+ //
142+ // Defaulted to true by config wiring; left as the Go zero value
143+ // (false) when Pipeline is constructed directly, so unit tests
144+ // with no LLM can opt out by simply not setting it.
145+ TOCEnabled bool
146+
147+ // TOCModel overrides the LLM model used by the TOC builder.
148+ // Empty inherits SummaryModel (which itself falls back to the
149+ // client default).
150+ TOCModel string
151+
152+ // TOCConcurrency caps parallel LLM calls during the TOC
153+ // verification phase. Default: 4.
154+ TOCConcurrency int
155+
156+ // TOCCheckPages bounds the leading prefix the detector scans
157+ // for a table of contents. Default: 20.
158+ TOCCheckPages int
159+
137160 // globalLLMSem is the lazily-initialized shared semaphore enforcing
138161 // GlobalLLMConcurrency. nil means "no global cap" — callers fall back
139162 // to per-stage limits only.
@@ -168,6 +191,12 @@ func NewPipeline(p Pipeline) *Pipeline {
168191 if p .HyDEConcurrency <= 0 {
169192 p .HyDEConcurrency = 4
170193 }
194+ if p .TOCConcurrency <= 0 {
195+ p .TOCConcurrency = 4
196+ }
197+ if p .TOCCheckPages <= 0 {
198+ p .TOCCheckPages = 20
199+ }
171200 // Default the global cap to a value that comfortably exceeds the
172201 // sum of the two default per-stage caps (4 + 4 = 8) while leaving
173202 // some headroom — but stays well below typical provider per-tenant
@@ -265,13 +294,131 @@ func (p *Pipeline) Run(ctx context.Context, pl Payload) error {
265294 }
266295 log .Info ("ingest: summarize+hyde complete" , "elapsed" , time .Since (stageStart ))
267296
297+ // LLM-built TOC tree (PageIndex-style). PDF-only because it
298+ // relies on the parser's PageStart/PageEnd attribution to
299+ // reconstruct per-page text. Non-fatal: a builder failure
300+ // leaves documents.toc_tree NULL and the document remains
301+ // fully retrievable via the sections tree above.
302+ if p .TOCEnabled && pl .ContentType == "application/pdf" {
303+ if err := p .runTOCBuilder (ctx , pl .DocumentID , parsed , log ); err != nil {
304+ log .Warn ("ingest: toc-builder failed; falling back to NULL toc_tree" , "err" , err )
305+ }
306+ }
307+
268308 if err := p .DB .SetDocumentStatus (ctx , pl .DocumentID , db .StatusReady , "" ); err != nil {
269309 return err
270310 }
271311 log .Info ("ingest: ready" )
272312 return nil
273313}
274314
315+ // runTOCBuilder assembles per-page text from the parsed PDF, runs
316+ // the LLM-driven TOC builder over it, and persists the result.
317+ // Returns an error only on a transport-level builder failure or a
318+ // JSON-marshal blip; the caller logs and continues either way.
319+ //
320+ // A nil-result (no usable nodes) is treated as success and writes
321+ // SQL NULL to documents.toc_tree (which is the column's default,
322+ // so this is also the no-op).
323+ func (p * Pipeline ) runTOCBuilder (ctx context.Context , docID tree.DocumentID , parsed * parser.ParsedDoc , log * slog.Logger ) error {
324+ pages := assemblePagesFromSections (parsed .Sections )
325+ if len (pages ) == 0 {
326+ log .Info ("ingest: toc-builder skipped; no per-page text available" )
327+ return nil
328+ }
329+ model := p .TOCModel
330+ if model == "" {
331+ model = p .SummaryModel
332+ }
333+ builder := & TOCBuilder {
334+ LLM : p .LLM ,
335+ Model : model ,
336+ Concurrency : p .TOCConcurrency ,
337+ TOCCheckPages : p .TOCCheckPages ,
338+ }
339+ nodes , usage , err := builder .Build (ctx , pages )
340+ if err != nil {
341+ return err
342+ }
343+ log .Info ("ingest: toc-builder done" ,
344+ "top_level_nodes" , len (nodes ),
345+ "llm_calls" , usage .LLMCalls ,
346+ "input_tokens" , usage .InputTokens ,
347+ "output_tokens" , usage .OutputTokens ,
348+ )
349+ if len (nodes ) == 0 {
350+ return nil
351+ }
352+ treeJSON , err := json .Marshal (nodes )
353+ if err != nil {
354+ return fmt .Errorf ("marshal toc tree: %w" , err )
355+ }
356+ if err := p .DB .UpdateDocumentTOCTree (ctx , docID , treeJSON ); err != nil {
357+ return fmt .Errorf ("persist toc tree: %w" , err )
358+ }
359+ return nil
360+ }
361+
362+ // assemblePagesFromSections groups the parsed sections' text by
363+ // their PageStart, producing PageText entries the TOC builder can
364+ // reason over. Sections that span multiple pages collapse onto
365+ // their starting page — perfect page reconstruction would need
366+ // raw glyph-level coordinates the parser doesn't currently
367+ // surface, but the title-on-claimed-page heuristic still works
368+ // because section starts (where the LLM looks for titles) live
369+ // on PageStart.
370+ //
371+ // Sections with PageStart == 0 are skipped (the parser couldn't
372+ // place them) so the builder never sees ambiguous page numbers.
373+ func assemblePagesFromSections (secs []parser.Section ) []PageText {
374+ pageText := map [int ]* strings.Builder {}
375+ pages := []int {}
376+ var walk func ([]parser.Section )
377+ walk = func (ss []parser.Section ) {
378+ for _ , s := range ss {
379+ if s .PageStart > 0 {
380+ b , ok := pageText [s .PageStart ]
381+ if ! ok {
382+ b = & strings.Builder {}
383+ pageText [s .PageStart ] = b
384+ pages = append (pages , s .PageStart )
385+ }
386+ if title := strings .TrimSpace (s .Title ); title != "" {
387+ if b .Len () > 0 {
388+ b .WriteByte ('\n' )
389+ }
390+ b .WriteString (title )
391+ b .WriteByte ('\n' )
392+ }
393+ if body := strings .TrimSpace (s .Content ); body != "" {
394+ b .WriteString (body )
395+ b .WriteByte ('\n' )
396+ }
397+ }
398+ walk (s .Children )
399+ }
400+ }
401+ walk (secs )
402+ // Sort the page-number index in place.
403+ sortIntsAscending (pages )
404+ out := make ([]PageText , 0 , len (pages ))
405+ for _ , p := range pages {
406+ out = append (out , PageText {PageNumber : p , Text : pageText [p ].String ()})
407+ }
408+ return out
409+ }
410+
411+ // sortIntsAscending sorts a slice of ints in place. Insertion sort
412+ // is fine here — pages slice is typically a few hundred items
413+ // at most.
414+ func sortIntsAscending (xs []int ) {
415+ for i := 1 ; i < len (xs ); i ++ {
416+ for j := i ; j > 0 && xs [j - 1 ] > xs [j ]; j -- {
417+ xs [j - 1 ], xs [j ] = xs [j ], xs [j - 1 ]
418+ }
419+ }
420+ }
421+
275422// runParallelStages runs summarize and HyDE concurrently, returning each
276423// stage's error independently so callers can log them separately. A nil
277424// hydeFn skips the HyDE stage (returns nil for hydeErr).
0 commit comments