Skip to content

Commit 61a01fe

Browse files
tae2089claude
andcommitted
Make pg_trgm fuzzy indexable, precise, and corpus-scoped
Address code-review findings on the fuzzy fallback: - Use the index-accelerated `<%` operator (rewritten to `%>`, which gin_trgm_ops supports) instead of `word_similarity(?, col) >= const`, which can never use the trigram indexes. Confirmed via EXPLAIN that the operator form yields a Bitmap Index Scan. - Fire fuzzy only on a total exact-FTS miss, not whenever exact underfills the limit, so precise queries are no longer diluted with loose matches. - Scope fuzzy to nodes that have a search_documents row (JOIN), keeping it within the same corpus and kind mix as the exact path. - Set the word_similarity threshold via transaction-local set_config so the operator cutoff matches the tuned value; log real failures at Warn while still degrading gracefully, and do not log on context cancellation. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent a610422 commit 61a01fe

1 file changed

Lines changed: 50 additions & 49 deletions

File tree

internal/store/search/postgres.go

Lines changed: 50 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"context"
66
"fmt"
77
"log/slog"
8+
"strconv"
89

910
"gorm.io/gorm"
1011

@@ -175,9 +176,11 @@ func (p *PostgresBackend) Query(ctx context.Context, db *gorm.DB, query string,
175176
}
176177

177178
// A zero-hit tsquery is not terminal: a misspelled query legitimately matches nothing
178-
// exactly, and the pg_trgm fuzzy supplement below can still surface candidates.
179+
// exactly, and the pg_trgm fuzzy supplement can still surface candidates. Fuzzy fires
180+
// only on a total exact miss so it never dilutes the precision of queries that did match.
179181
if len(rows) == 0 {
180-
return p.appendFuzzyMatches(ctx, db, query, ns, nil, limit), nil
182+
fuzzy := p.appendFuzzyMatches(ctx, db, query, ns, limit)
183+
return promoteExactNameMatch(fuzzy, query), nil
181184
}
182185

183186
nodeIDs := make([]uint, len(rows))
@@ -209,76 +212,74 @@ func (p *PostgresBackend) Query(ctx context.Context, db *gorm.DB, query string,
209212
}
210213
}
211214

212-
// When exact FTS underfills the requested limit, supplement with typo-tolerant
213-
// trigram matches on symbol names so misspelled queries still return candidates.
214-
if len(result) < limit {
215-
result = p.appendFuzzyMatches(ctx, db, query, ns, result, limit)
216-
}
217-
218215
result = promoteExactNameMatch(result, query)
219216
return result, nil
220217
}
221218

222-
// appendFuzzyMatches supplements exact-FTS results with pg_trgm similarity matches on
223-
// symbol names, ordered by similarity. It is best-effort: when pg_trgm is unavailable the
224-
// query errors and the exact results are returned unchanged.
219+
// appendFuzzyMatches returns up to limit pg_trgm fuzzy matches on symbol names for queries
220+
// that produced no exact FTS hit (typo tolerance). It is best-effort: if pg_trgm is
221+
// unavailable the query errors and an empty result is returned so exact search still works.
225222
// @intent add typo tolerance to Postgres search without letting a missing extension break exact search.
226-
func (p *PostgresBackend) appendFuzzyMatches(ctx context.Context, db *gorm.DB, query, ns string, existing []model.Node, limit int) []model.Node {
227-
remaining := limit - len(existing)
228-
if remaining <= 0 {
229-
return existing
230-
}
231-
seen := make(map[uint]struct{}, len(existing))
232-
for _, n := range existing {
233-
seen[n.ID] = struct{}{}
223+
// @domainRule only nodes that also have a search_documents row are eligible, so fuzzy stays
224+
// within the same corpus and kind mix as the exact FTS path.
225+
func (p *PostgresBackend) appendFuzzyMatches(ctx context.Context, db *gorm.DB, query, ns string, limit int) []model.Node {
226+
if limit <= 0 {
227+
return nil
234228
}
235229

236-
// word_similarity (not similarity) is used so a short query matches the best-fitting
237-
// extent of a longer symbol name; plain similarity penalizes length differences and
238-
// misses typos like "authentcate" inside "AuthenticateUser".
230+
// The `<%` operator is index-accelerated by the gin_trgm_ops indexes on name/qualified_name
231+
// (a functional `word_similarity(...) >= const` filter cannot use them); its cutoff is the
232+
// session-local word_similarity_threshold, set here so the operator and our tuned threshold
233+
// agree. Ordering uses the functional form over the already-filtered small set.
239234
var rows []resultRow
240235
fuzzySQL := `
241-
SELECT id AS node_id
242-
FROM nodes
243-
WHERE namespace = ?
244-
AND (word_similarity(?, name) >= ? OR word_similarity(?, qualified_name) >= ?)
245-
ORDER BY GREATEST(word_similarity(?, name), word_similarity(?, qualified_name)) DESC
236+
SELECT n.id AS node_id
237+
FROM nodes n
238+
JOIN search_documents sd ON sd.node_id = n.id AND sd.namespace = n.namespace
239+
WHERE n.namespace = ?
240+
AND (? <% n.name OR ? <% n.qualified_name)
241+
ORDER BY GREATEST(word_similarity(?, n.name), word_similarity(?, n.qualified_name)) DESC
246242
LIMIT ?`
247-
if err := db.WithContext(ctx).Raw(fuzzySQL, ns, query, fuzzyWordSimilarityThreshold, query, fuzzyWordSimilarityThreshold, query, query, remaining+len(existing)).Scan(&rows).Error; err != nil {
248-
slog.Debug("pg_trgm fuzzy supplement skipped", trace.SlogError(err))
249-
return existing
250-
}
251-
252-
fuzzyIDs := make([]uint, 0, len(rows))
253-
for _, r := range rows {
254-
if _, ok := seen[r.NodeID]; ok {
255-
continue
243+
err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
244+
if e := tx.Exec(`SELECT set_config('pg_trgm.word_similarity_threshold', ?, true)`,
245+
strconv.FormatFloat(fuzzyWordSimilarityThreshold, 'f', -1, 64)).Error; e != nil {
246+
return e
256247
}
257-
seen[r.NodeID] = struct{}{}
258-
fuzzyIDs = append(fuzzyIDs, r.NodeID)
248+
return tx.Raw(fuzzySQL, ns, query, query, query, query, limit).Scan(&rows).Error
249+
})
250+
if err != nil {
251+
// Context cancellation is the caller's concern, not a fuzzy fault; only surface real errors.
252+
if ctx.Err() == nil {
253+
slog.Warn("pg_trgm fuzzy supplement failed", trace.SlogError(err))
254+
}
255+
return nil
259256
}
260-
if len(fuzzyIDs) == 0 {
261-
return existing
257+
if len(rows) == 0 {
258+
return nil
262259
}
263260

261+
nodeIDs := make([]uint, len(rows))
262+
for i, r := range rows {
263+
nodeIDs[i] = r.NodeID
264+
}
264265
var nodes []model.Node
265-
if err := db.WithContext(ctx).Where("id IN ?", fuzzyIDs).Where("namespace = ?", ns).Find(&nodes).Error; err != nil {
266-
slog.Debug("pg_trgm fuzzy node load skipped", trace.SlogError(err))
267-
return existing
266+
if err := db.WithContext(ctx).Where("id IN ?", nodeIDs).Where("namespace = ?", ns).Find(&nodes).Error; err != nil {
267+
if ctx.Err() == nil {
268+
slog.Warn("pg_trgm fuzzy node load failed", trace.SlogError(err))
269+
}
270+
return nil
268271
}
269272
byID := make(map[uint]model.Node, len(nodes))
270273
for _, n := range nodes {
271274
byID[n.ID] = n
272275
}
273-
for _, id := range fuzzyIDs {
276+
ordered := make([]model.Node, 0, len(nodeIDs))
277+
for _, id := range nodeIDs {
274278
if n, ok := byID[id]; ok && n.ID != 0 {
275-
existing = append(existing, n)
276-
if len(existing) >= limit {
277-
break
278-
}
279+
ordered = append(ordered, n)
279280
}
280281
}
281-
return existing
282+
return ordered
282283
}
283284

284285
var _ Backend = (*PostgresBackend)(nil)

0 commit comments

Comments
 (0)