fix(tree+ingest): show all sections + drop bad bytes before LLM

Three related bugs from inspecting the GINA PDF upload that reached
'ready' with an empty ToC:

1. buildTree silently dropped multiple top-level sections. It picked
   the first row with NULL parent_id as the tree root and ignored
   the rest, so BuildView only walked one subtree. This PDF had three
   top-level siblings (the title splits across pages) and ~280 sections
   under the other two were invisible to /toc. Now we wrap multiple
   top-level sections in a synthetic root so BuildView walks them all.

2. Section content was written to storage without UTF-8 validation.
   PDFs with CID-mapped fonts and no ToUnicode CMap leak raw glyph
   IDs into extracted text, which the Gemini SDK then rejects with
   "google.ai.generativelanguage.v1beta.Part.text contains invalid
   UTF-8". cleanForLLM strips the bad bytes and disallowed C0 controls
   (keeping \t \n \r) at storage-write time AND on the read-back path
   that feeds the LLM.

3. CountSections + plumbing through HandleGetDocument so the dashboard
   metadata panel can render the actual section count instead of a
   blank field. The doc list endpoint still doesn't include it (one
   query per row is a footgun for large libraries) — only the detail
   endpoint.

Author: hallelx2

pkg/db/sections.go

@@ -62,6 +62,22 @@ func (p *Pool) UpdateSectionSummary(ctx context.Context, id tree.SectionID, summ
 	return mapErr(err)
 }
 
+// CountSections returns the number of sections persisted for a
+// document. Used by the dashboard metadata panel; not org-scoped on
+// purpose — pair with a prior GetDocument(orgID) check to ensure the
+// caller is allowed to see the document at all.
+func (p *Pool) CountSections(ctx context.Context, docID tree.DocumentID) (int, error) {
+	var n int
+	err := p.QueryRow(ctx,
+		`SELECT count(*) FROM sections WHERE document_id = $1`,
+		string(docID),
+	).Scan(&n)
+	if err != nil {
+		return 0, mapErr(err)
+	}
+	return n, nil
+}
+
 // GetSection fetches a single section, scoped to an org via JOIN on
 // documents.org_id. Cross-org reads return ErrNotFound rather than
 // the row, so section IDs from other tenants can't be probed.
@@ -214,13 +230,16 @@ func buildTree(doc *Document, rows []Section) *tree.Tree {
 		}
 	}
 
-	var root *tree.Section
+	// Collect every section whose parent_id is empty — these are
+	// "top-level" sections. Older code picked just the first one as
+	// the tree root and silently dropped the other siblings + their
+	// subtrees. Now we wrap them in a synthetic root so callers see
+	// the whole document.
+	var topLevel []*tree.Section
 	for i := range rows {
 		s := byID[rows[i].ID]
 		if s.ParentID == "" {
-			if root == nil {
-				root = s
-			}
+			topLevel = append(topLevel, s)
 			continue
 		}
 		parent, ok := byID[s.ParentID]
@@ -230,6 +249,25 @@ func buildTree(doc *Document, rows []Section) *tree.Tree {
 		parent.Children = append(parent.Children, s)
 	}
 
+	var root *tree.Section
+	switch len(topLevel) {
+	case 0:
+		root = nil // empty doc
+	case 1:
+		root = topLevel[0]
+	default:
+		// Multiple top-level sections — wrap in a synthetic root so
+		// BuildView walks all of them. The synthetic root carries the
+		// document's title and an empty ID so consumers can distinguish
+		// it from a real section.
+		root = &tree.Section{
+			ID:       "",
+			ParentID: "",
+			Title:    doc.Title,
+			Children: topLevel,
+		}
+	}
+
 	return &tree.Tree{
 		DocumentID: doc.ID,
 		Title:      doc.Title,

pkg/ingest/ingest.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"unicode/utf8"
 
 	"github.com/google/uuid"
 	"golang.org/x/sync/errgroup"
@@ -159,12 +160,17 @@ func (p *Pipeline) persistTree(ctx context.Context, docID tree.DocumentID, doc *
 			id := tree.SectionID("sec_" + uuid.New().String())
 			contentKey := path.Join("documents", string(docID), "sections", string(id)+".txt")
 
-			if strings.TrimSpace(s.Content) != "" {
+			// Strip invalid UTF-8 / disallowed control chars at storage
+			// time so we never persist bytes the LLM SDKs would reject
+			// later. PDFs with CID-mapped fonts and no ToUnicode CMap
+			// leak raw glyph IDs into extracted text.
+			cleanedContent := cleanForLLM(s.Content)
+			if strings.TrimSpace(cleanedContent) != "" {
 				if err := p.Storage.Put(ctx, contentKey,
-					bytes.NewReader([]byte(s.Content)),
+					bytes.NewReader([]byte(cleanedContent)),
 					storage.Metadata{
 						ContentType: "text/plain; charset=utf-8",
-						Size:        int64(len(s.Content)),
+						Size:        int64(len(cleanedContent)),
 					}); err != nil {
 					return fmt.Errorf("store section %s: %w", id, err)
 				}
@@ -176,9 +182,9 @@ func (p *Pipeline) persistTree(ctx context.Context, docID tree.DocumentID, doc *
 				ParentID:   parent,
 				Ordinal:    i,
 				Depth:      depth,
-				Title:      s.Title,
+				Title:      cleanForLLM(s.Title),
 				ContentRef: contentKey,
-				TokenCount: approxTokens(s.Content),
+				TokenCount: approxTokens(cleanedContent),
 				Metadata:   s.Metadata,
 			}); err != nil {
 				return err
@@ -296,7 +302,7 @@ func (p *Pipeline) summaryFor(ctx context.Context, s db.Section, kids []db.Secti
 	if len(kids) == 0 {
 		// Leaf: fetch the stored text.
 		if s.ContentRef == "" {
-			return s.Title, nil
+			return cleanForLLM(s.Title), nil
 		}
 		rc, _, err := p.Storage.Get(ctx, s.ContentRef)
 		if err != nil {
@@ -307,7 +313,7 @@ func (p *Pipeline) summaryFor(ctx context.Context, s db.Section, kids []db.Secti
 		if err != nil {
 			return "", err
 		}
-		body = string(raw)
+		body = cleanForLLM(string(raw))
 	} else {
 		// Internal: compose from children's titles so we have SOMETHING to
 		// summarize even without bringing all children content into memory.
@@ -327,7 +333,7 @@ func (p *Pipeline) summaryFor(ctx context.Context, s db.Section, kids []db.Secti
 			{Role: llmgate.RoleSystem, Content: "You write short, factual section summaries. One sentence, no preamble, no quotes."},
 			{Role: llmgate.RoleUser, Content: fmt.Sprintf(
 				"Summarize this section titled %q in a single sentence (max 40 words):\n\n%s",
-				s.Title, body)},
+				cleanForLLM(s.Title), body)},
 		},
 	})
 	if err != nil {
@@ -370,6 +376,46 @@ func (p *Pipeline) fail(ctx context.Context, id tree.DocumentID, stage string, c
 	}
 }
 
+// cleanForLLM strips invalid-UTF-8 bytes and a couple of control
+// characters that some LLM SDKs reject at the proto layer (the
+// gemini-go SDK is strict about this — it errors with
+// "google.ai.generativelanguage.v1beta.Part.text contains invalid UTF-8"
+// the moment any byte sequence isn't a complete UTF-8 codepoint).
+//
+// PDFs with custom CID-mapped fonts and no ToUnicode CMap leak raw
+// glyph IDs into our extracted text, which look like garbage bytes.
+// We drop them rather than fail the whole summarization.
+func cleanForLLM(s string) string {
+	if utf8.ValidString(s) && !hasBadControlChars(s) {
+		return s
+	}
+	var b strings.Builder
+	b.Grow(len(s))
+	for i := 0; i < len(s); {
+		r, size := utf8.DecodeRuneInString(s[i:])
+		i += size
+		if r == utf8.RuneError && size == 1 {
+			b.WriteRune('�')
+			continue
+		}
+		// Drop NUL + most C0 control chars; keep tab/newline/CR.
+		if r < 0x20 && r != '\t' && r != '\n' && r != '\r' {
+			continue
+		}
+		b.WriteRune(r)
+	}
+	return b.String()
+}
+
+func hasBadControlChars(s string) bool {
+	for _, r := range s {
+		if r < 0x20 && r != '\t' && r != '\n' && r != '\r' {
+			return true
+		}
+	}
+	return false
+}
+
 // isLikelyMojibakeTitle returns true when s shows the doubled-glyph
 // signature of two-layer PDFs (an overlay watermark drawn over real
 // text at the same Y coordinate, so chars from both layers interleave

pkg/ingest/mojibake_test.go

@@ -1,6 +1,37 @@
 package ingest
 
-import "testing"
+import (
+	"testing"
+	"unicode/utf8"
+)
+
+func TestCleanForLLM(t *testing.T) {
+	cases := []struct {
+		in   string
+		want string
+	}{
+		// Valid text passes through unchanged.
+		{"Hello world", "Hello world"},
+		{"Section title — with em dash", "Section title — with em dash"},
+		{"Multi-line\nbody\nstays intact", "Multi-line\nbody\nstays intact"},
+		// Invalid bytes get replaced with U+FFFD.
+		{"bad \xff byte", "bad � byte"},
+		// NUL + most C0 controls get dropped; tab/newline/CR are kept.
+		{"keep\ttabs\nand\rcrs but drop\x00nul", "keep\ttabs\nand\rcrs but dropnul"},
+		{"text with bell\x07char", "text with bellchar"},
+		// Empty input survives.
+		{"", ""},
+	}
+	for _, tc := range cases {
+		got := cleanForLLM(tc.in)
+		if got != tc.want {
+			t.Errorf("cleanForLLM(%q) = %q, want %q", tc.in, got, tc.want)
+		}
+		if !utf8.ValidString(got) {
+			t.Errorf("cleanForLLM(%q) returned invalid UTF-8", tc.in)
+		}
+	}
+}
 
 func TestIsLikelyMojibakeTitle(t *testing.T) {
 	cases := []struct {