security: fix silent data corruption in three edge cases

index_writer.go

@@ -4,6 +4,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
+	"math"
 	"os"
 
 	"github.com/cespare/xxhash/v2"
@@ -54,6 +55,12 @@ type indexWriter struct {
 func newIndexWriter(path string, cfg *buildConfig, numBlocks uint32, algo blockBuilder) (*indexWriter, error) {
 	// Compute variable section sizes
 	userMetadataLen := len(cfg.userMetadata)
+	// The length is written to disk as a uint32, so reject anything larger.
+	// Comparing via uint64 keeps this compiling on 32-bit builds, where the
+	// math.MaxUint32 constant doesn't fit in an int.
+	if uint64(userMetadataLen) > math.MaxUint32 {
+		return nil, fmt.Errorf("user metadata size %d exceeds maximum of %d bytes", userMetadataLen, uint64(math.MaxUint32))
+	}
 	algoConfigLen := algo.GlobalConfigSize()
 
 	// Estimate total file size for pre-allocation

internal/bijection/builder.go

@@ -246,7 +246,10 @@ func (bb *Builder) MaxMetadataSizeForCurrentBlock() int {
 func (bb *Builder) encodeSeparatedMetadataInto(dst []byte) (int, error) {
 	// Step 1: Encode Elias-Fano cumulative counts and compute EF checkpoints
 	var cp checkpoints
-	efData := bb.encodeEFWithCheckpoints(&cp, bucketsPerBlock)
+	efData, err := bb.encodeEFWithCheckpoints(&cp, bucketsPerBlock)
+	if err != nil {
+		return 0, err
+	}
 
 	// Step 2: Encode SeedStream and compute Seed checkpoints
 	bb.seedEncoder.resetEncoder()
@@ -406,7 +409,7 @@ func (bb *Builder) resolveFallbackSeed(bucketIdx int, subBucket uint8) uint32 {
 }
 
 // encodeEFWithCheckpoints encodes Elias-Fano and computes EF checkpoints.
-func (bb *Builder) encodeEFWithCheckpoints(cp *checkpoints, numBuckets int) []byte {
+func (bb *Builder) encodeEFWithCheckpoints(cp *checkpoints, numBuckets int) ([]byte, error) {
 	efData := encodeEliasFanoInto(bb.cumulative, uint32(bb.keysInBlock), &bb.efBuffer)
 
 	lowBits := computeEFLowBits(numBuckets, bb.keysInBlock)
@@ -423,11 +426,18 @@ func (bb *Builder) encodeEFWithCheckpoints(cp *checkpoints, numBuckets int) []by
 			highPart = int(prevCumulative) >> lowBits
 		}
 
-		bitPos := min(bucketIdx+highPart, math.MaxUint16)
+		// Checkpoints are stored as uint16. This shouldn't overflow for valid
+		// blocks, but if it ever does, error out instead of clamping to a wrong
+		// offset (which would silently corrupt query results). Mirrors the
+		// seedBitPos check in encodeSeedsWithCheckpoints.
+		bitPos := bucketIdx + highPart
+		if bitPos > math.MaxUint16 {
+			return nil, fmt.Errorf("bijection: EF bit position overflow: %d exceeds uint16 max", bitPos)
+		}
 		cp.efBitPos[checkpointIdx] = uint16(bitPos)
 	}
 
-	return efData
+	return efData, nil
 }
 
 // encodeSeedsWithCheckpoints encodes seeds and computes checkpoints.

internal/encoding/encoding.go

@@ -6,6 +6,16 @@ package encoding
 
 import "unsafe"
 
+func init() {
+	// Fail loudly on big-endian CPUs rather than writing silently corrupt files
+	// (see the package doc above). The probe is 0x0102; on little-endian its first
+	// byte in memory is 0x02.
+	var probe uint16 = 0x0102
+	if *(*byte)(unsafe.Pointer(&probe)) != 0x02 {
+		panic("encoding: package requires a little-endian architecture")
+	}
+}
+
 // WriteEntry packs a fingerprint and payload into a little-endian entry of
 // entrySize bytes at position pos in the buffer starting at basePtr.
 // The entry is stored as: fp | (payload << fpShift), where fpShift is