diff --git a/index_writer.go b/index_writer.go index 636013f..44e036c 100644 --- a/index_writer.go +++ b/index_writer.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "errors" "fmt" + "math" "os" "github.com/cespare/xxhash/v2" @@ -54,6 +55,12 @@ type indexWriter struct { func newIndexWriter(path string, cfg *buildConfig, numBlocks uint32, algo blockBuilder) (*indexWriter, error) { // Compute variable section sizes userMetadataLen := len(cfg.userMetadata) + // The length is written to disk as a uint32, so reject anything larger. + // Comparing via uint64 keeps this compiling on 32-bit builds, where the + // math.MaxUint32 constant doesn't fit in an int. + if uint64(userMetadataLen) > math.MaxUint32 { + return nil, fmt.Errorf("user metadata size %d exceeds maximum of %d bytes", userMetadataLen, uint64(math.MaxUint32)) + } algoConfigLen := algo.GlobalConfigSize() // Estimate total file size for pre-allocation diff --git a/internal/bijection/builder.go b/internal/bijection/builder.go index a43d48e..103b117 100644 --- a/internal/bijection/builder.go +++ b/internal/bijection/builder.go @@ -246,7 +246,10 @@ func (bb *Builder) MaxMetadataSizeForCurrentBlock() int { func (bb *Builder) encodeSeparatedMetadataInto(dst []byte) (int, error) { // Step 1: Encode Elias-Fano cumulative counts and compute EF checkpoints var cp checkpoints - efData := bb.encodeEFWithCheckpoints(&cp, bucketsPerBlock) + efData, err := bb.encodeEFWithCheckpoints(&cp, bucketsPerBlock) + if err != nil { + return 0, err + } // Step 2: Encode SeedStream and compute Seed checkpoints bb.seedEncoder.resetEncoder() @@ -406,7 +409,7 @@ func (bb *Builder) resolveFallbackSeed(bucketIdx int, subBucket uint8) uint32 { } // encodeEFWithCheckpoints encodes Elias-Fano and computes EF checkpoints. -func (bb *Builder) encodeEFWithCheckpoints(cp *checkpoints, numBuckets int) []byte { +func (bb *Builder) encodeEFWithCheckpoints(cp *checkpoints, numBuckets int) ([]byte, error) { efData := encodeEliasFanoInto(bb.cumulative, uint32(bb.keysInBlock), &bb.efBuffer) lowBits := computeEFLowBits(numBuckets, bb.keysInBlock) @@ -423,11 +426,18 @@ func (bb *Builder) encodeEFWithCheckpoints(cp *checkpoints, numBuckets int) []by highPart = int(prevCumulative) >> lowBits } - bitPos := min(bucketIdx+highPart, math.MaxUint16) + // Checkpoints are stored as uint16. This shouldn't overflow for valid + // blocks, but if it ever does, error out instead of clamping to a wrong + // offset (which would silently corrupt query results). Mirrors the + // seedBitPos check in encodeSeedsWithCheckpoints. + bitPos := bucketIdx + highPart + if bitPos > math.MaxUint16 { + return nil, fmt.Errorf("bijection: EF bit position overflow: %d exceeds uint16 max", bitPos) + } cp.efBitPos[checkpointIdx] = uint16(bitPos) } - return efData + return efData, nil } // encodeSeedsWithCheckpoints encodes seeds and computes checkpoints. diff --git a/internal/encoding/encoding.go b/internal/encoding/encoding.go index f28e0ab..e02ed82 100644 --- a/internal/encoding/encoding.go +++ b/internal/encoding/encoding.go @@ -6,6 +6,16 @@ package encoding import "unsafe" +func init() { + // Fail loudly on big-endian CPUs rather than writing silently corrupt files + // (see the package doc above). The probe is 0x0102; on little-endian its first + // byte in memory is 0x02. + var probe uint16 = 0x0102 + if *(*byte)(unsafe.Pointer(&probe)) != 0x02 { + panic("encoding: package requires a little-endian architecture") + } +} + // WriteEntry packs a fingerprint and payload into a little-endian entry of // entrySize bytes at position pos in the buffer starting at basePtr. // The entry is stored as: fp | (payload << fpShift), where fpShift is