perf(bm25): replace facet storage with compact direct Badger KV encoding

Replace the facet-based BM25 storage (~40-50 bytes/posting) with compact
varint-encoded binary blobs stored as direct Badger KV entries (~4-6
bytes/posting, ~10x reduction). Add bm25_score pseudo-predicate for
variable-based score ordering following the similar_to pattern.

- Add posting/bm25enc package for compact binary encode/decode
- Rewrite write path in posting/index.go for direct Badger KV
- Add bm25Writes buffer to LocalCache with read-your-own-writes
- Flush BM25 blobs in CommitToDisk with BitBM25Data UserMeta
- Rewrite read path in worker/task.go with direct blob decoding
- Add bm25_score pseudo-predicate in query/query.go
- Add score ordering integration tests

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: shaunpatterson

posting/bm25enc/bm25enc.go

@@ -0,0 +1,147 @@
+/*
+ * SPDX-FileCopyrightText: © 2017-2025 Istari Digital, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+// Package bm25enc provides compact binary encoding for BM25 index data.
+//
+// Two types of lists share the same format:
+//   - Term posting lists: (UID, term-frequency) pairs
+//   - Document length lists: (UID, doc-length) pairs
+//
+// Binary format:
+//
+//	Header:
+//	  [4 bytes] uint32 big-endian: entry count
+//	Entries (sorted ascending by UID):
+//	  [varint] UID delta from previous (first entry is absolute)
+//	  [varint] value (TF or doclen)
+package bm25enc
+
+import (
+	"encoding/binary"
+	"sort"
+)
+
+// Entry represents a single (UID, Value) pair in a BM25 posting list.
+type Entry struct {
+	UID   uint64
+	Value uint32
+}
+
+// Encode encodes a sorted slice of entries into the compact binary format.
+// Entries must be sorted by UID ascending. Returns nil for empty input.
+func Encode(entries []Entry) []byte {
+	if len(entries) == 0 {
+		return nil
+	}
+
+	// Pre-allocate: 4 header + ~6 bytes per entry is a reasonable estimate.
+	buf := make([]byte, 4, 4+len(entries)*6)
+	binary.BigEndian.PutUint32(buf, uint32(len(entries)))
+
+	var tmp [binary.MaxVarintLen64]byte
+	var prevUID uint64
+	for _, e := range entries {
+		delta := e.UID - prevUID
+		n := binary.PutUvarint(tmp[:], delta)
+		buf = append(buf, tmp[:n]...)
+		n = binary.PutUvarint(tmp[:], uint64(e.Value))
+		buf = append(buf, tmp[:n]...)
+		prevUID = e.UID
+	}
+	return buf
+}
+
+// Decode decodes the binary format into a sorted slice of entries.
+// Returns nil for nil/empty input.
+func Decode(data []byte) []Entry {
+	if len(data) < 4 {
+		return nil
+	}
+	count := binary.BigEndian.Uint32(data[:4])
+	if count == 0 {
+		return nil
+	}
+
+	entries := make([]Entry, 0, count)
+	pos := 4
+	var prevUID uint64
+	for i := uint32(0); i < count; i++ {
+		delta, n := binary.Uvarint(data[pos:])
+		if n <= 0 {
+			break
+		}
+		pos += n
+
+		val, n := binary.Uvarint(data[pos:])
+		if n <= 0 {
+			break
+		}
+		pos += n
+
+		uid := prevUID + delta
+		entries = append(entries, Entry{UID: uid, Value: uint32(val)})
+		prevUID = uid
+	}
+	return entries
+}
+
+// Upsert inserts or updates the entry for uid in a sorted entries slice.
+// Returns the new sorted slice.
+func Upsert(entries []Entry, uid uint64, value uint32) []Entry {
+	i := sort.Search(len(entries), func(i int) bool { return entries[i].UID >= uid })
+	if i < len(entries) && entries[i].UID == uid {
+		entries[i].Value = value
+		return entries
+	}
+	// Insert at position i.
+	entries = append(entries, Entry{})
+	copy(entries[i+1:], entries[i:])
+	entries[i] = Entry{UID: uid, Value: value}
+	return entries
+}
+
+// Remove removes the entry for uid from a sorted entries slice.
+// Returns the new slice (may be shorter).
+func Remove(entries []Entry, uid uint64) []Entry {
+	i := sort.Search(len(entries), func(i int) bool { return entries[i].UID >= uid })
+	if i < len(entries) && entries[i].UID == uid {
+		return append(entries[:i], entries[i+1:]...)
+	}
+	return entries
+}
+
+// Search returns the value for uid using binary search, and whether it was found.
+func Search(entries []Entry, uid uint64) (uint32, bool) {
+	i := sort.Search(len(entries), func(i int) bool { return entries[i].UID >= uid })
+	if i < len(entries) && entries[i].UID == uid {
+		return entries[i].Value, true
+	}
+	return 0, false
+}
+
+// UIDs extracts just the UIDs from entries as a uint64 slice.
+func UIDs(entries []Entry) []uint64 {
+	uids := make([]uint64, len(entries))
+	for i, e := range entries {
+		uids[i] = e.UID
+	}
+	return uids
+}
+
+// EncodeStats encodes BM25 corpus statistics (docCount, totalTerms) as 16 bytes.
+func EncodeStats(docCount, totalTerms uint64) []byte {
+	buf := make([]byte, 16)
+	binary.BigEndian.PutUint64(buf[0:8], docCount)
+	binary.BigEndian.PutUint64(buf[8:16], totalTerms)
+	return buf
+}
+
+// DecodeStats decodes BM25 corpus statistics. Returns (0,0) for invalid input.
+func DecodeStats(data []byte) (docCount, totalTerms uint64) {
+	if len(data) != 16 {
+		return 0, 0
+	}
+	return binary.BigEndian.Uint64(data[0:8]), binary.BigEndian.Uint64(data[8:16])
+}

posting/bm25enc/bm25enc_test.go

@@ -0,0 +1,132 @@
+/*
+ * SPDX-FileCopyrightText: © 2017-2025 Istari Digital, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package bm25enc
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestRoundtrip(t *testing.T) {
+	entries := []Entry{
+		{UID: 1, Value: 3},
+		{UID: 5, Value: 1},
+		{UID: 100, Value: 7},
+		{UID: 200, Value: 2},
+	}
+	data := Encode(entries)
+	got := Decode(data)
+	require.Equal(t, entries, got)
+}
+
+func TestRoundtripEmpty(t *testing.T) {
+	require.Nil(t, Encode(nil))
+	require.Nil(t, Encode([]Entry{}))
+	require.Nil(t, Decode(nil))
+	require.Nil(t, Decode([]byte{}))
+	require.Nil(t, Decode([]byte{0, 0, 0, 0})) // count=0
+}
+
+func TestRoundtripSingle(t *testing.T) {
+	entries := []Entry{{UID: 42, Value: 10}}
+	got := Decode(Encode(entries))
+	require.Equal(t, entries, got)
+}
+
+func TestRoundtripLargeUIDs(t *testing.T) {
+	entries := []Entry{
+		{UID: 1<<40 + 1, Value: 1},
+		{UID: 1<<40 + 1000, Value: 5},
+		{UID: 1<<50 + 999, Value: 99},
+	}
+	got := Decode(Encode(entries))
+	require.Equal(t, entries, got)
+}
+
+func TestUpsertNew(t *testing.T) {
+	entries := []Entry{{UID: 1, Value: 3}, {UID: 5, Value: 1}}
+	entries = Upsert(entries, 3, 7)
+	require.Equal(t, []Entry{{UID: 1, Value: 3}, {UID: 3, Value: 7}, {UID: 5, Value: 1}}, entries)
+}
+
+func TestUpsertExisting(t *testing.T) {
+	entries := []Entry{{UID: 1, Value: 3}, {UID: 5, Value: 1}}
+	entries = Upsert(entries, 5, 99)
+	require.Equal(t, []Entry{{UID: 1, Value: 3}, {UID: 5, Value: 99}}, entries)
+}
+
+func TestUpsertEmpty(t *testing.T) {
+	var entries []Entry
+	entries = Upsert(entries, 10, 5)
+	require.Equal(t, []Entry{{UID: 10, Value: 5}}, entries)
+}
+
+func TestRemove(t *testing.T) {
+	entries := []Entry{{UID: 1, Value: 3}, {UID: 5, Value: 1}, {UID: 10, Value: 2}}
+	entries = Remove(entries, 5)
+	require.Equal(t, []Entry{{UID: 1, Value: 3}, {UID: 10, Value: 2}}, entries)
+}
+
+func TestRemoveNotFound(t *testing.T) {
+	entries := []Entry{{UID: 1, Value: 3}, {UID: 5, Value: 1}}
+	entries = Remove(entries, 99)
+	require.Equal(t, []Entry{{UID: 1, Value: 3}, {UID: 5, Value: 1}}, entries)
+}
+
+func TestSearch(t *testing.T) {
+	entries := []Entry{{UID: 1, Value: 3}, {UID: 5, Value: 1}, {UID: 100, Value: 7}}
+	v, ok := Search(entries, 5)
+	require.True(t, ok)
+	require.Equal(t, uint32(1), v)
+
+	_, ok = Search(entries, 50)
+	require.False(t, ok)
+}
+
+func TestUIDs(t *testing.T) {
+	entries := []Entry{{UID: 1, Value: 3}, {UID: 5, Value: 1}, {UID: 100, Value: 7}}
+	require.Equal(t, []uint64{1, 5, 100}, UIDs(entries))
+}
+
+func TestStatsRoundtrip(t *testing.T) {
+	data := EncodeStats(12345, 98765)
+	dc, tt := DecodeStats(data)
+	require.Equal(t, uint64(12345), dc)
+	require.Equal(t, uint64(98765), tt)
+}
+
+func TestStatsInvalid(t *testing.T) {
+	dc, tt := DecodeStats(nil)
+	require.Zero(t, dc)
+	require.Zero(t, tt)
+	dc, tt = DecodeStats([]byte{1, 2, 3})
+	require.Zero(t, dc)
+	require.Zero(t, tt)
+}
+
+func BenchmarkEncode(b *testing.B) {
+	entries := make([]Entry, 10000)
+	for i := range entries {
+		entries[i] = Entry{UID: uint64(i*3 + 1), Value: uint32(i % 100)}
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		Encode(entries)
+	}
+}
+
+func BenchmarkDecode(b *testing.B) {
+	entries := make([]Entry, 10000)
+	for i := range entries {
+		entries[i] = Entry{UID: uint64(i*3 + 1), Value: uint32(i % 100)}
+	}
+	data := Encode(entries)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		Decode(data)
+	}
+}