fix(parser): make the leaf-section cap reduce any tree shape

The cap only merged ADJACENT leaf siblings, but the real 10-K explosion
is hundreds of SINGLE-LEAF PARENTS (heading -> one body leaf) that have
no adjacent leaf-sibling pairs at all — so the cap silently did nothing
and a 92-page filing sailed past the 400 cap at 463-1465 leaves, each
costing a summarize + HyDE + multi-axis LLM call (the throughput killer
for full ingest).

Fix it in two phases:
  1. collapseSingleLeafParents flattens every heading -> lone-leaf chain
     (bottom-up, so deep chains fold in one pass) so the formerly
     only-children become adjacent leaf siblings. Count is unchanged; the
     parent absorbs the child's content + page range and becomes the leaf.
  2. The existing smallest-first adjacent-pair merge then reduces the
     count to the cap, with a defensive single-leaf-parent collapse for
     any pair a table leaf blocked.

The top-level sections are wrapped under a synthetic root so the merge
step — which shrinks a sibling list by rewriting parent.Children — can
shrink the TOP-level list too (a bare slice parameter would not
propagate the shrink back). This is what makes the invariant hold for a
flat list of single-leaf parents.

Invariant: for any tree with > N mergeable leaves, capLeafSections drives
the leaf count to <= N. Content is always preserved (concatenated,
page ranges unioned) and table sections (Metadata["table"]=="true") are
never merged or collapsed. Nothing is disabled — the full section tree is
still produced; it's just bounded.

Tests: 1000 single-leaf-parents -> <= 400 with no content loss (the case
the old cap failed and that let 1465 through), deep heading->subheading->
body chains, a mixed flat/single-leaf/multi-child tree, and a
table-protection test asserting table leaves survive verbatim.

Author: hallelx2

pkg/parser/cap_test.go

@@ -15,6 +15,29 @@ func leafTree(n, size int) []Section {
 	return []Section{{Level: 1, Title: "parent", Children: kids}}
 }
 
+// singleLeafParentTree builds n top-level "heading" sections, each an
+// internal node with EXACTLY ONE leaf child carrying `size` chars. This
+// is the real 10-K explosion shape: hundreds of heading -> one-body-leaf
+// parents with NO adjacent leaf-sibling pairs anywhere. The pre-fix cap
+// (adjacent-siblings only) could not reduce this tree at all.
+func singleLeafParentTree(n, size int) []Section {
+	parents := make([]Section, n)
+	for i := range parents {
+		parents[i] = Section{
+			Level: 1,
+			Title: "heading",
+			Children: []Section{{
+				Level:     2,
+				Title:     "body",
+				Content:   strings.Repeat("y", size),
+				PageStart: i + 1,
+				PageEnd:   i + 1,
+			}},
+		}
+	}
+	return parents
+}
+
 func TestCapLeafSections_MergesDownToCap(t *testing.T) {
 	tree := leafTree(1000, 50)
 	if got := countLeafSections(tree); got != 1000 {
@@ -81,6 +104,148 @@ func TestCapLeafSections_MergesSmallestFirst(t *testing.T) {
 	}
 }
 
+// TestCapLeafSections_SingleLeafParentsReduce is the regression test for
+// the bug that let a 92-page 10-K through at 463-1465 leaves: a tree made
+// entirely of single-leaf parents (heading -> one body leaf) has no
+// adjacent leaf-sibling pairs, so the old adjacent-only merge did nothing.
+// The fix must collapse those chains and merge down to the cap, losing no
+// content.
+func TestCapLeafSections_SingleLeafParentsReduce(t *testing.T) {
+	tree := singleLeafParentTree(1000, 30)
+	if got := countLeafSections(tree); got != 1000 {
+		t.Fatalf("setup: countLeafSections = %d, want 1000", got)
+	}
+	orig := totalContentLen(tree) // 1000 * 30
+
+	capped := capLeafSections(tree, 400)
+
+	if got := countLeafSections(capped); got > 400 {
+		t.Errorf("after cap: %d leaves, want <= 400 (the bug let 1465 through)", got)
+	}
+	// No content lost. Collapses/merges insert at most a "\n\n" (2 chars)
+	// per fold; with < 1000 folds total the upper bound is generous.
+	if got := totalContentLen(capped); got < orig || got > orig+2*1000 {
+		t.Errorf("content not preserved: got %d chars, want in [%d, %d]", got, orig, orig+2*1000)
+	}
+}
+
+// TestCapLeafSections_DeepSingleLeafChains exercises heading -> subheading
+// -> body chains (depth-3 single-leaf parents). The bottom-up collapse
+// must fold the whole chain so the bodies become mergeable siblings.
+func TestCapLeafSections_DeepSingleLeafChains(t *testing.T) {
+	const n = 600
+	roots := make([]Section, n)
+	for i := range roots {
+		roots[i] = Section{Level: 1, Title: "h1", Children: []Section{{
+			Level: 2, Title: "h2", Children: []Section{{
+				Level: 3, Title: "body", Content: strings.Repeat("z", 20),
+				PageStart: i + 1, PageEnd: i + 1,
+			}},
+		}}}
+	}
+	if got := countLeafSections(roots); got != n {
+		t.Fatalf("setup: countLeafSections = %d, want %d", got, n)
+	}
+	orig := totalContentLen(roots)
+
+	capped := capLeafSections(roots, 400)
+	if got := countLeafSections(capped); got > 400 {
+		t.Errorf("after cap: %d leaves, want <= 400", got)
+	}
+	if got := totalContentLen(capped); got < orig {
+		t.Errorf("content shrank: got %d chars, want >= %d", got, orig)
+	}
+}
+
+// TestCapLeafSections_MixedShapeReducesToCap throws a tree that mixes a
+// flat sibling list, single-leaf parents, and a multi-child branch at the
+// cap. The invariant is unconditional: > N mergeable leaves must come
+// down to <= N regardless of shape.
+func TestCapLeafSections_MixedShapeReducesToCap(t *testing.T) {
+	tree := []Section{
+		{Level: 1, Title: "flat", Children: func() []Section {
+			out := make([]Section, 300)
+			for i := range out {
+				out[i] = Section{Level: 2, Title: "f", Content: "ff", PageStart: i + 1, PageEnd: i + 1}
+			}
+			return out
+		}()},
+	}
+	// Append 300 single-leaf parents as additional top-level nodes.
+	tree = append(tree, singleLeafParentTree(300, 10)...)
+
+	if got := countLeafSections(tree); got != 600 {
+		t.Fatalf("setup: countLeafSections = %d, want 600", got)
+	}
+	capped := capLeafSections(tree, 100)
+	if got := countLeafSections(capped); got > 100 {
+		t.Errorf("after cap: %d leaves, want <= 100", got)
+	}
+}
+
+// TestCapLeafSections_NeverMergesTableSections asserts that table leaves
+// (Metadata["table"]="true") are preserved verbatim and never merged or
+// collapsed, even under a cap small enough that everything else must
+// merge around them.
+func TestCapLeafSections_NeverMergesTableSections(t *testing.T) {
+	mkTable := func(page int, content string) Section {
+		return Section{
+			Level: 1, Title: "Table", Content: content,
+			PageStart: page, PageEnd: page,
+			Metadata: map[string]string{"table": "true"},
+		}
+	}
+	// Three tables interleaved with prose leaves under one parent, plus a
+	// pile of single-leaf-parent prose so we're well over the cap.
+	kids := []Section{
+		{Level: 2, Title: "p1", Content: "prose-a"},
+		mkTable(1, "| A | B |\n| --- | --- |\n| 1 | 2 |"),
+		{Level: 2, Title: "p2", Content: "prose-b"},
+		mkTable(2, "| C | D |\n| --- | --- |\n| 3 | 4 |"),
+		{Level: 2, Title: "p3", Content: "prose-c"},
+		mkTable(3, "| E | F |\n| --- | --- |\n| 5 | 6 |"),
+	}
+	tree := []Section{{Level: 1, Title: "parent", Children: kids}}
+	tree = append(tree, singleLeafParentTree(200, 10)...)
+
+	capped := capLeafSections(tree, 5)
+
+	// Every original table's content must still be present, verbatim, in
+	// some leaf.
+	tableContents := []string{
+		"| A | B |\n| --- | --- |\n| 1 | 2 |",
+		"| C | D |\n| --- | --- |\n| 3 | 4 |",
+		"| E | F |\n| --- | --- |\n| 5 | 6 |",
+	}
+	var leaves []Section
+	var collect func([]Section)
+	collect = func(ss []Section) {
+		for i := range ss {
+			if len(ss[i].Children) == 0 {
+				leaves = append(leaves, ss[i])
+			} else {
+				collect(ss[i].Children)
+			}
+		}
+	}
+	collect(capped)
+
+	for _, want := range tableContents {
+		found := false
+		for _, l := range leaves {
+			// A table must survive as its OWN leaf — its content present and
+			// the table metadata intact (i.e. it wasn't merged into prose).
+			if l.Content == want && l.Metadata["table"] == "true" {
+				found = true
+				break
+			}
+		}
+		if !found {
+			t.Errorf("table content %q was merged away or lost its metadata", want)
+		}
+	}
+}
+
 func totalContentLen(sections []Section) int {
 	n := 0
 	for i := range sections {

pkg/parser/pdf.go

@@ -605,34 +605,158 @@ func countLeafSections(sections []Section) int {
 	return n
 }
 
-// capLeafSections enforces a ceiling on the total leaf-section count.
-// While the count exceeds maxLeaves it repeatedly merges the two
-// smallest ADJACENT leaf siblings under whichever parent currently has
-// the most leaf children — so the runaway byte-split sections collapse
-// back first, while genuinely distinct top-level sections are left
-// alone. maxLeaves <= 0 disables the cap.
+// capLeafSections enforces a ceiling on the total leaf-section count so a
+// pathological PDF can't shatter into thousands of tiny leaves (each of
+// which costs a summarize + HyDE + multi-axis LLM call at ingest, which
+// is what throttles/stalls the full pipeline). maxLeaves <= 0 disables
+// the cap; a tree already at or under the cap is returned untouched.
 //
-// Merged leaves concatenate their content (blank-line separated), keep
-// the first sibling's title, and union their page ranges. Table
-// sections are attached AFTER this pass (attachTableSections), so their
-// numeric content is never merged away.
+// The reduction is robust to ANY tree shape, which is the fix for the
+// real 10-K explosion: that document is not a flat list of adjacent
+// leaves but hundreds of SINGLE-LEAF PARENTS (heading -> one body leaf),
+// which have no adjacent leaf-sibling pairs at all — so the old
+// adjacent-only merge silently did nothing and a 92-page filing sailed
+// past the cap at 463-1465 leaves. We fix it in two phases:
+//
+//  1. collapseSingleLeafParents flattens every heading -> lone-leaf chain
+//     so the formerly-only-children become adjacent leaf SIBLINGS. This
+//     restructures the tree without changing the leaf count (the parent
+//     absorbs the child and itself becomes the leaf).
+//
+//  2. The merge loop then repeatedly merges the smallest adjacent leaf
+//     pair until the count is back under the cap (a defensive collapse
+//     step covers any pair a table leaf blocked).
+//
+// The invariant: for any tree with > maxLeaves MERGEABLE leaves,
+// capLeafSections drives countLeafSections to <= maxLeaves. The only
+// leaves it will not merge are table sections (Metadata["table"]=="true")
+// — those carry distinct numeric content and must survive verbatim. In
+// the normal parse flow table sections are attached AFTER this pass
+// (attachTableSections), so the cap never sees them; the guard is
+// defensive for any caller that pre-attaches tables.
+//
+// Merged/collapsed leaves concatenate their content (blank-line
+// separated), keep the parent/first-sibling title, and union their page
+// ranges, so no body text is ever dropped.
 func capLeafSections(sections []Section, maxLeaves int) []Section {
 	if maxLeaves <= 0 {
 		return sections
 	}
-	// Guard against pathological loops: at most one merge per excess leaf.
-	for guard := 0; countLeafSections(sections) > maxLeaves && guard < 100000; guard++ {
-		if !mergeOneSmallestAdjacentLeafPair(sections) {
-			break // no mergeable adjacent leaf pair anywhere
+	if countLeafSections(sections) <= maxLeaves {
+		return sections // already under the cap — leave structure untouched
+	}
+
+	// Wrap the top-level sections under a synthetic root. The merge step
+	// shrinks a sibling list in place (it rewrites parent.Children), which
+	// only propagates back to the caller when the mutated list is a struct
+	// FIELD, not a bare slice parameter. The runaway shape we're fixing —
+	// hundreds of single-leaf PARENTS — needs the TOP-level list to shrink,
+	// so we make the top level a nested list (root.Children) and operate on
+	// that; the wrapper itself (length 1) never changes. We return
+	// root.Children at the end.
+	root := []Section{{Children: sections}}
+
+	// Phase 1: flatten single-leaf-parent chains so only-children become
+	// mergeable siblings. Count is unchanged; structure is normalised.
+	root[0].Children = collapseSingleLeafParents(root[0].Children)
+
+	// Phase 2: merge adjacent leaf pairs (smallest first) until under cap.
+	// Each merge removes one leaf and each fallback collapse removes one
+	// internal node — both strictly monotonic — so the loop is bounded;
+	// the guard is belt-and-braces against a logic regression.
+	for guard := 0; countLeafSections(root) > maxLeaves && guard < 1000000; guard++ {
+		if mergeOneSmallestAdjacentLeafPair(root) {
+			continue
+		}
+		// No adjacent mergeable pair left. Try to free one up by
+		// collapsing a remaining single-leaf parent; if that's impossible
+		// too, every remaining leaf is unmergeable (e.g. table sections)
+		// and we stop — there is nothing left we're permitted to merge.
+		if !collapseOneSingleLeafParent(&root) {
+			break
+		}
+	}
+	return root[0].Children
+}
+
+// isTableLeaf reports whether s is a table section that must never be
+// merged or collapsed (its cell content is distinct numeric data).
+func isTableLeaf(s *Section) bool {
+	return s.Metadata != nil && s.Metadata["table"] == "true"
+}
+
+// collapseSingleLeafParents recursively flattens every parent that has
+// exactly one child which is a (non-table) leaf: the parent absorbs the
+// child's content + page range and itself becomes the leaf. Chains
+// (heading -> subheading -> body) collapse fully in one bottom-up pass.
+//
+// Leaf count is preserved (one internal node + one leaf become one leaf);
+// the point is purely to turn only-children into adjacent siblings so the
+// adjacent-pair merge can then reduce the count. Returns the rewritten
+// slice.
+func collapseSingleLeafParents(sections []Section) []Section {
+	for i := range sections {
+		s := &sections[i]
+		if len(s.Children) == 0 {
+			continue
+		}
+		// Bottom-up: collapse within the children first so a chain folds
+		// from the leaf upward.
+		s.Children = collapseSingleLeafParents(s.Children)
+		if len(s.Children) == 1 && len(s.Children[0].Children) == 0 && !isTableLeaf(&s.Children[0]) {
+			absorbChildIntoParent(s, s.Children[0])
+			s.Children = nil
 		}
 	}
 	return sections
 }
 
+// collapseOneSingleLeafParent finds the first parent in the tree with
+// exactly one (non-table) leaf child and collapses it (parent absorbs the
+// leaf, becomes a leaf). Returns false when no such parent exists. Used
+// as a defensive fallback inside the merge loop when an adjacent pair was
+// blocked by a table leaf; the bulk flattening is done up front by
+// collapseSingleLeafParents.
+func collapseOneSingleLeafParent(sections *[]Section) bool {
+	s := *sections
+	for i := range s {
+		n := &s[i]
+		if len(n.Children) == 1 && len(n.Children[0].Children) == 0 && !isTableLeaf(&n.Children[0]) {
+			absorbChildIntoParent(n, n.Children[0])
+			n.Children = nil
+			return true
+		}
+		if len(n.Children) > 0 {
+			if collapseOneSingleLeafParent(&n.Children) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// absorbChildIntoParent folds a leaf child's content and page range up
+// into its parent. The parent keeps its own title (the heading) and
+// gains the child's body; an empty parent body is replaced outright so we
+// don't prefix a stray separator. Page ranges union (min start, max end).
+func absorbChildIntoParent(parent *Section, child Section) {
+	switch {
+	case strings.TrimSpace(parent.Content) == "":
+		parent.Content = child.Content
+	case strings.TrimSpace(child.Content) != "":
+		parent.Content = parent.Content + "\n\n" + child.Content
+	}
+	parent.PageStart = minNonZero(parent.PageStart, child.PageStart)
+	if child.PageEnd > parent.PageEnd {
+		parent.PageEnd = child.PageEnd
+	}
+}
+
 // mergeOneSmallestAdjacentLeafPair finds the adjacent leaf-sibling pair
 // with the smallest combined content length anywhere in the tree and
-// merges it in place. Returns false when no sibling list has two
-// adjacent leaves to merge.
+// merges it in place. Only pairs where BOTH siblings are NON-table leaves
+// are eligible — table sections are never merged. Returns false when no
+// sibling list has two adjacent mergeable leaves.
 func mergeOneSmallestAdjacentLeafPair(sections []Section) bool {
 	bestList := (*[]Section)(nil)
 	bestIdx := -1
@@ -642,7 +766,8 @@ func mergeOneSmallestAdjacentLeafPair(sections []Section) bool {
 	walk = func(list *[]Section) {
 		s := *list
 		for i := 0; i+1 < len(s); i++ {
-			if len(s[i].Children) == 0 && len(s[i+1].Children) == 0 {
+			if len(s[i].Children) == 0 && len(s[i+1].Children) == 0 &&
+				!isTableLeaf(&s[i]) && !isTableLeaf(&s[i+1]) {
 				size := len(s[i].Content) + len(s[i+1].Content)
 				if bestSize < 0 || size < bestSize {
 					bestSize, bestList, bestIdx = size, list, i