feat(parser): table-fidelity loaders + XLSX/CSV; reframe pageindex citations; llmgate v0.3.0

Loaders: extract shared buildSections/deriveTitle (markdown/html/docx); DOCX & HTML tables serialize to Markdown grids; new XLSX (stdlib) + CSV loaders; DOCX outlineLvl heading fallback + footnotes; smarter plain-text heading splitting.

Retrieval: reframe pageindex citation discipline from 'ideally ONE' to minimal-sufficient-set (widen 2+ carve-out, add two-range few-shot example).

Deps: bump llmgate to v0.3.0 (GLM-4.6 pricing so per-query cost != 0; native tool calling; Usage.Priced). Wire pricing.WarnFunc in cmd/engine and cmd/server.

Author: hallelx2

cmd/engine/main.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"github.com/hallelx2/llmgate"
+	"github.com/hallelx2/llmgate/pricing"
 	"github.com/hallelx2/llmgate/provider/anthropic"
 	"github.com/hallelx2/llmgate/provider/gemini"
 	"github.com/hallelx2/llmgate/provider/openai"
@@ -61,6 +62,12 @@ func run() error {
 		"retrieval_strategy", cfg.Retrieval.Strategy,
 	)
 
+	// Surface any model with no price-book entry: its cost reads $0, which
+	// would otherwise masquerade as "free" in usage/benchmark accounting.
+	pricing.WarnFunc = func(model string) {
+		logger.Warn("llm model not in price book; cost reported as 0", "model", model)
+	}
+
 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
 	defer stop()
 

cmd/server/main.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/hallelx2/llmgate"
+	"github.com/hallelx2/llmgate/pricing"
 	"github.com/hallelx2/llmgate/provider/anthropic"
 	"github.com/hallelx2/llmgate/provider/gemini"
 	"github.com/hallelx2/llmgate/provider/openai"
@@ -80,6 +81,12 @@ func run() error {
 		"retrieval_strategy", cfg.Engine.Retrieval.Strategy,
 	)
 
+	// Surface any model with no price-book entry: its cost reads $0, which
+	// would otherwise masquerade as "free" in usage/benchmark accounting.
+	pricing.WarnFunc = func(model string) {
+		logger.Warn("llm model not in price book; cost reported as 0", "model", model)
+	}
+
 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
 	defer stop()
 

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/aws/smithy-go v1.25.0
 	github.com/go-chi/chi/v5 v5.2.5
 	github.com/google/uuid v1.6.0
-	github.com/hallelx2/llmgate v0.2.0
+	github.com/hallelx2/llmgate v0.3.0
 	github.com/hallelx2/pdftable v0.3.1
 	github.com/hibiken/asynq v0.26.0
 	github.com/jackc/pgx/v5 v5.9.2

go.sum

@@ -132,8 +132,8 @@ github.com/googleapis/gax-go/v2 v2.21.0 h1:h45NjjzEO3faG9Lg/cFrBh2PgegVVgzqKzuZl
 github.com/googleapis/gax-go/v2 v2.21.0/go.mod h1:But/NJU6TnZsrLai/xBAQLLz+Hc7fHZJt/hsCz3Fih4=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+uBhcekkmy4IkffJww=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90=
-github.com/hallelx2/llmgate v0.2.0 h1:x/LNCeHUPZpafn2IXi+LqpnZa7TtEQdLVlpkkJTlzBI=
-github.com/hallelx2/llmgate v0.2.0/go.mod h1:MK2Ol/5CIweTQ2/9eSiTJ5g/KSSuobNZL9TD3s57JxY=
+github.com/hallelx2/llmgate v0.3.0 h1:OLm2L41WFAqE2ka64RUdLirXmVRbvbY8zu7N3NSkfS8=
+github.com/hallelx2/llmgate v0.3.0/go.mod h1:MK2Ol/5CIweTQ2/9eSiTJ5g/KSSuobNZL9TD3s57JxY=
 github.com/hallelx2/pdftable v0.3.1 h1:Uqe+9G8s9jrGYwxk8dEMXBCB+SlzvWPmW0Ze5863W1I=
 github.com/hallelx2/pdftable v0.3.1/go.mod h1:pxNlc4D43wjzis7M6EfgQZvHOsQ4okggm+xqUu+OokI=
 github.com/hhrutter/lzw v1.0.0 h1:laL89Llp86W3rRs83LvKbwYRx6INE8gDn0XNb1oXtm0=

pkg/ingest/ingest.go

@@ -1116,6 +1116,8 @@ func DefaultRegistry() *parser.Registry {
 		parser.NewMarkdown(),
 		parser.NewHTML(),
 		parser.NewDOCX(),
+		parser.NewXLSX(),
+		parser.NewCSV(),
 		parser.NewPDF(),
 		parser.NewText(),
 	)
@@ -1150,6 +1152,8 @@ func RegistryFromIngestParams(opts *parser.TableOpts, maxSections int, parseTime
 		parser.NewMarkdown(),
 		parser.NewHTML(),
 		parser.NewDOCX(),
+		parser.NewXLSX(),
+		parser.NewCSV(),
 		parser.NewPDFWithConfig(opts, maxSections, parseTimeout),
 		parser.NewText(),
 	)

pkg/parser/csv.go

@@ -0,0 +1,59 @@
+package parser
+
+import (
+	"context"
+	"encoding/csv"
+	"io"
+	"strings"
+)
+
+// CSV parses comma-separated values into a single section rendered as a
+// Markdown grid, so the table structure (and each value's column) is
+// preserved for the LLM rather than collapsed into prose. The first row is
+// treated as the header.
+type CSV struct{}
+
+// NewCSV returns a new CSV parser.
+func NewCSV() *CSV { return &CSV{} }
+
+// Name implements Parser.
+func (*CSV) Name() string { return "csv" }
+
+// Accepts implements Parser.
+func (*CSV) Accepts(contentType, filename string) bool {
+	switch contentType {
+	case "text/csv", "application/csv":
+		return true
+	}
+	return HasExt(filename, ".csv")
+}
+
+// Parse implements Parser.
+func (*CSV) Parse(_ context.Context, r io.Reader) (*ParsedDoc, error) {
+	cr := csv.NewReader(r)
+	cr.FieldsPerRecord = -1 // tolerate ragged rows
+	cr.LazyQuotes = true
+
+	records, err := cr.ReadAll()
+	if err != nil {
+		return nil, err
+	}
+
+	title := "Data"
+	if len(records) > 0 && len(records[0]) > 0 {
+		// Use the first header cell as a hint if it reads like a label.
+		if h := strings.TrimSpace(records[0][0]); h != "" && len(h) <= 60 {
+			title = h
+		}
+	}
+
+	content := renderTableMarkdown(records)
+	return &ParsedDoc{
+		Title: title,
+		Sections: []Section{{
+			Level:   1,
+			Title:   "Data",
+			Content: content,
+		}},
+	}, nil
+}