pkg/parser: split oversized leaf sections so cover pages become retrievable

Filing cover pages (and any other long, mixed-topic leaf section) produce
one 2-3k-char blob under a generic title like "3M COMPANY" — mixing
registration tables, addresses, IRS IDs, contact info. A single summary
can't cover all those topics, so retrieval picks unrelated "long-term
debt" sections instead of the one that actually holds the answer.

Add chunkOversizedLeaves: any LEAF section whose Content exceeds 2400
chars is replaced by a parent (title preserved) with smaller children at
the next level. Children are sized around 900 chars and split at word
boundaries. The chunk title prefers a natural colon-terminated header
within the first 80 chars ("Securities registered pursuant to Section
12(b) of the Act:") when available — exactly the pattern in filings —
otherwise the first ~60 chars trimmed at a word boundary, falling back
to "<parent title> — part N".

Internal nodes are recursed into but never split (they're already
structured). Threshold deliberately high (2400) so most paper sub-
sections aren't affected; combined with the retrieval-friendly summary
prompt (previous commit), each chunk gets a topic-rich summary downstream
so the retrieval LLM can match it to specific questions.

Tests in chunk_test.go: oversized leaf gets split with the parent title
preserved + children at level+1; first chunk takes the colon-header
title; small sections are untouched; oversized leaves nested inside
internal nodes are still split.

Author: hallelx2

pkg/parser/chunk_test.go

@@ -0,0 +1,92 @@
+package parser
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestChunkOversizedLeavesSplits(t *testing.T) {
+	// 12 words per "sentence", 5 sentences ~ 60-65 words, ~360 chars; we want
+	// >2400 chars so build it from a longer paragraph + a colon-terminated header.
+	header := "Securities registered pursuant to Section 12(b) of the Act: "
+	long := strings.Repeat("alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu ", 60)
+	content := header + long
+	if len(content) <= leafChunkThreshold {
+		t.Fatalf("test setup: content must exceed threshold; got %d", len(content))
+	}
+	in := []Section{{Level: 1, Title: "3M COMPANY", Content: content}}
+
+	out := chunkOversizedLeaves(in)
+	if len(out) != 1 {
+		t.Fatalf("expected 1 top-level section, got %d", len(out))
+	}
+	parent := out[0]
+	if parent.Title != "3M COMPANY" {
+		t.Errorf("parent title should be preserved, got %q", parent.Title)
+	}
+	if parent.Content != "" {
+		t.Errorf("parent content should be cleared after splitting, got %d chars", len(parent.Content))
+	}
+	if len(parent.Children) < 2 {
+		t.Fatalf("expected multiple chunks, got %d", len(parent.Children))
+	}
+	// First chunk's title should use the colon-terminated header.
+	if !strings.HasPrefix(parent.Children[0].Title, "Securities registered pursuant to Section 12(b)") {
+		t.Errorf("first chunk title should come from the colon header, got %q", parent.Children[0].Title)
+	}
+	// Every chunk's content should be non-empty and well below the original.
+	for i, c := range parent.Children {
+		if c.Content == "" {
+			t.Errorf("chunk %d has empty content", i)
+		}
+		if len(c.Content) > leafChunkTarget*2 {
+			t.Errorf("chunk %d larger than expected: %d chars", i, len(c.Content))
+		}
+	}
+}
+
+func TestChunkOversizedLeavesLeavesSmallSectionsAlone(t *testing.T) {
+	in := []Section{
+		{Level: 1, Title: "Intro", Content: strings.Repeat("a b c d e f ", 50)},  // ~600 chars
+		{Level: 1, Title: "Methods", Content: strings.Repeat("x y z ", 200)},      // ~1200 chars
+	}
+	out := chunkOversizedLeaves(in)
+	if len(out) != 2 {
+		t.Fatalf("expected 2 sections preserved, got %d", len(out))
+	}
+	for i, s := range out {
+		if len(s.Children) != 0 {
+			t.Errorf("section %d was unexpectedly split into %d children", i, len(s.Children))
+		}
+	}
+}
+
+func TestChunkOversizedLeavesRecursesIntoInternals(t *testing.T) {
+	bigLeaf := Section{Level: 2, Title: "Detail", Content: strings.Repeat("the quick brown fox jumps over the lazy dog ", 100)}
+	parent := Section{Level: 1, Title: "Parent", Children: []Section{bigLeaf}}
+	out := chunkOversizedLeaves([]Section{parent})
+	if len(out) != 1 || len(out[0].Children) == 0 {
+		t.Fatalf("parent should be retained with chunked children, got %+v", out)
+	}
+	leaf := out[0].Children[0]
+	if leaf.Title != "Detail" {
+		t.Errorf("inner leaf title should be preserved, got %q", leaf.Title)
+	}
+	if len(leaf.Children) < 2 {
+		t.Errorf("inner leaf should have been chunked, has %d children", len(leaf.Children))
+	}
+}
+
+func TestDeriveChunkTitleColonHeader(t *testing.T) {
+	got := deriveChunkTitle("Securities registered pursuant to Section 12(b) of the Act: Title of each class ...", "fallback")
+	want := "Securities registered pursuant to Section 12(b) of the Act"
+	if got != want {
+		t.Errorf("colon-header title: got %q want %q", got, want)
+	}
+}
+
+func TestDeriveChunkTitleFallback(t *testing.T) {
+	if got := deriveChunkTitle("", "fb"); got != "fb" {
+		t.Errorf("empty chunk should fall back, got %q", got)
+	}
+}

pkg/parser/pdf.go

@@ -230,10 +230,118 @@ func (*PDF) Parse(_ context.Context, r io.Reader) (*ParsedDoc, error) {
 
 	return &ParsedDoc{
 		Title:    title,
-		Sections: rootSec.Children,
+		Sections: chunkOversizedLeaves(rootSec.Children),
 	}, nil
 }
 
+// Filing cover pages (and any other long, mixed-topic leaf) often produce one
+// 2-3k-char section under a generic title like "3M COMPANY", which mixes
+// registration tables, addresses, IRS IDs and contact info. A single summary
+// can't cover all those topics, so retrieval misses. Split such leaves into
+// smaller sub-sections at word boundaries; each sub-section then gets its own
+// title (from a natural colon-terminated header, e.g. "Securities registered
+// pursuant to Section 12(b) of the Act", or the first few words) and its own
+// summary downstream.
+const (
+	leafChunkThreshold = 2400 // chars; high enough to leave paper sub-sections alone
+	leafChunkTarget    = 900  // chars per chunk, give or take
+)
+
+// chunkOversizedLeaves splits any LEAF section whose content exceeds
+// leafChunkThreshold into smaller sub-sections. Internal nodes (sections with
+// children) are recursed into but never split — they're already structured.
+func chunkOversizedLeaves(sections []Section) []Section {
+	out := make([]Section, 0, len(sections))
+	for _, s := range sections {
+		if len(s.Children) > 0 {
+			s.Children = chunkOversizedLeaves(s.Children)
+			out = append(out, s)
+			continue
+		}
+		if len(s.Content) <= leafChunkThreshold {
+			out = append(out, s)
+			continue
+		}
+		pieces := splitContentByWords(s.Content, leafChunkTarget)
+		if len(pieces) <= 1 {
+			out = append(out, s)
+			continue
+		}
+		parent := Section{Level: s.Level, Title: s.Title}
+		for i, piece := range pieces {
+			fallback := fmt.Sprintf("%s — part %d", s.Title, i+1)
+			parent.Children = append(parent.Children, Section{
+				Level:   s.Level + 1,
+				Title:   deriveChunkTitle(piece, fallback),
+				Content: piece,
+			})
+		}
+		out = append(out, parent)
+	}
+	return out
+}
+
+// splitContentByWords breaks a long string into pieces near target size at
+// word boundaries. The last piece may be smaller; pieces are never midword.
+func splitContentByWords(s string, target int) []string {
+	s = strings.TrimSpace(s)
+	if target < 200 {
+		target = 200
+	}
+	slack := target / 4
+	if len(s) <= target+slack {
+		return []string{s}
+	}
+	var chunks []string
+	for len(s) > 0 {
+		if len(s) <= target+slack {
+			chunks = append(chunks, strings.TrimSpace(s))
+			break
+		}
+		upper := target + slack
+		if upper > len(s) {
+			upper = len(s)
+		}
+		cut := strings.LastIndex(s[:upper], " ")
+		if cut < target/2 {
+			cut = upper // no good break: hard-cut at upper bound
+		}
+		chunks = append(chunks, strings.TrimSpace(s[:cut]))
+		s = strings.TrimSpace(s[cut:])
+	}
+	return chunks
+}
+
+// deriveChunkTitle picks a readable label for a content chunk. Prefers a
+// phrase ending in ":" within the first ~80 chars (filings use these as
+// natural sub-headers, e.g. "Securities registered pursuant to Section 12(b)
+// of the Act:"); otherwise takes the first ~60 chars trimmed at a word
+// boundary. Falls back to the supplied default when degenerate.
+func deriveChunkTitle(chunk, fallback string) string {
+	s := strings.TrimSpace(chunk)
+	if s == "" {
+		return fallback
+	}
+	if i := strings.Index(s, ":"); i > 0 && i < 80 {
+		candidate := strings.TrimSpace(s[:i])
+		if len(strings.Fields(candidate)) >= 2 {
+			return candidate
+		}
+	}
+	if len(s) <= 60 {
+		return strings.TrimRight(s, " ,;.:")
+	}
+	cut := strings.LastIndex(s[:60], " ")
+	if cut < 30 {
+		cut = 60
+	}
+	t := strings.TrimRight(strings.TrimSpace(s[:cut]), " ,;.:")
+	if t == "" {
+		return fallback
+	}
+	return t
+}
+
 type pdfRow struct {
 	page     int
 	fontSize float64