Skip to content

Commit 6bd613a

Browse files
dfa1claude
dfa1
and
claude
committed
feat(encoding): add fastlanes.for decoder
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent dd85298 commit 6bd613a

4 files changed

Lines changed: 265 additions & 2 deletions

File tree

‎TODO.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
- `dict` — dictionary encoding for low-cardinality columns
2222
- `pcodec` — float compression
2323

24-
- [ ] **#7a Fix `fastlanes.bitpacked` — spec-compliant rewrite**
24+
- [x] **#7a Fix `fastlanes.bitpacked` — spec-compliant rewrite**
2525
- Root cause: current code guesses format by metadata byte size (9 = Java, 2 = JNI). Wrong.
2626
The spec always uses protobuf metadata regardless of writer origin.
2727
- **Spec** (from `encodings/fastlanes/src/bitpacking/vtable/mod.rs`):
@@ -35,7 +35,7 @@
3535
- Step 5: update `BitpackedCodecTest` for round-trip with spec-compliant metadata
3636
- Reference: `spiraldb/vortex` `encodings/fastlanes/src/bitpacking/`, `spiraldb/fastlanes-rs` `src/bitpacking.rs` + `src/macros.rs`
3737

38-
- [ ] **#7b Implement `fastlanes.for` decoder**
38+
- [x] **#7b Implement `fastlanes.for` decoder**
3939
- **Spec** (from `encodings/fastlanes/src/for/vtable/mod.rs`):
4040
- Metadata: raw `ScalarValue` protobuf bytes (the reference/minimum value; no wrapper message)
4141
- Child slot 0: encoded array (typically `fastlanes.bitpacked` residuals)

‎core/src/main/java/io/github/dfa1/vortex/encoding/FrameOfReferenceCodec.java‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
package io.github.dfa1.vortex.encoding;
2+
3+
import com.google.protobuf.InvalidProtocolBufferException;
4+
import dev.vortex.proto.ScalarProtos;
5+
import io.github.dfa1.vortex.core.DType;
6+
import io.github.dfa1.vortex.core.PType;
7+
8+
import java.io.IOException;
9+
import java.lang.foreign.MemorySegment;
10+
import java.lang.foreign.ValueLayout;
11+
import java.nio.ByteBuffer;
12+
import java.nio.ByteOrder;
13+
14+
/// Decoder for {@code fastlanes.for} (Frame of Reference).
15+
///
16+
/// <p>Metadata: raw {@code ScalarValue} protobuf bytes — the reference (minimum) value.
17+
/// Child slot 0: encoded residuals array (same dtype as parent, typically bitpacked).
18+
/// Decode: {@code output[i] = encoded[i] + reference} (wrapping arithmetic).
19+
public final class FrameOfReferenceCodec implements Decoder {
20+
21+
@Override
22+
public String encodingId() {
23+
return "fastlanes.for";
24+
}
25+
26+
@Override
27+
public Array decode(DecodeContext ctx) {
28+
ByteBuffer rawMeta = ctx.metadata();
29+
if (rawMeta == null || !rawMeta.hasRemaining()) {
30+
throw new IllegalStateException("fastlanes.for: missing metadata");
31+
}
32+
byte[] metaBytes = new byte[rawMeta.remaining()];
33+
rawMeta.duplicate().get(metaBytes);
34+
35+
ScalarProtos.ScalarValue scalar;
36+
try {
37+
scalar = ScalarProtos.ScalarValue.parseFrom(metaBytes);
38+
} catch (InvalidProtocolBufferException e) {
39+
throw new IllegalStateException("fastlanes.for: invalid metadata", e);
40+
}
41+
42+
Array encoded;
43+
try {
44+
encoded = ctx.decodeChild(0);
45+
} catch (IOException e) {
46+
throw new IllegalStateException("fastlanes.for: failed to decode child", e);
47+
}
48+
49+
if (!(ctx.dtype() instanceof DType.Primitive p)) {
50+
throw new IllegalStateException("fastlanes.for: expected primitive dtype, got " + ctx.dtype());
51+
}
52+
53+
long ref = referenceValue(scalar);
54+
if (ref == 0L) {
55+
return encoded;
56+
}
57+
58+
MemorySegment src = encoded.buffer(0);
59+
long n = ctx.rowCount();
60+
MemorySegment dst = applyReference(src, n, p.ptype(), ref);
61+
return new Array(ctx.dtype(), n, new MemorySegment[]{dst}, new Array[0], ArrayStats.empty());
62+
}
63+
64+
private static long referenceValue(ScalarProtos.ScalarValue scalar) {
65+
return switch (scalar.getKindCase()) {
66+
case INT64_VALUE -> scalar.getInt64Value();
67+
case UINT64_VALUE -> scalar.getUint64Value();
68+
case KIND_NOT_SET -> 0L;
69+
default -> throw new IllegalStateException(
70+
"fastlanes.for: unexpected scalar kind " + scalar.getKindCase());
71+
};
72+
}
73+
74+
private static MemorySegment applyReference(MemorySegment src, long n, PType ptype, long ref) {
75+
int elemBytes = ptype.byteSize();
76+
byte[] bytes = new byte[(int) (n * elemBytes)];
77+
ByteBuffer dst = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN);
78+
switch (ptype) {
79+
case I8, U8 -> {
80+
for (long i = 0; i < n; i++) {
81+
byte v = src.get(ValueLayout.JAVA_BYTE, i);
82+
dst.put((byte) (v + (byte) ref));
83+
}
84+
}
85+
case I16, U16 -> {
86+
var layout = ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
87+
for (long i = 0; i < n; i++) {
88+
short v = src.get(layout, i * 2);
89+
dst.putShort((short) (v + (short) ref));
90+
}
91+
}
92+
case I32, U32 -> {
93+
var layout = ValueLayout.JAVA_INT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
94+
for (long i = 0; i < n; i++) {
95+
int v = src.get(layout, i * 4);
96+
dst.putInt(v + (int) ref);
97+
}
98+
}
99+
case I64, U64 -> {
100+
var layout = ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
101+
for (long i = 0; i < n; i++) {
102+
long v = src.get(layout, i * 8);
103+
dst.putLong(v + ref);
104+
}
105+
}
106+
default -> throw new UnsupportedOperationException(
107+
"fastlanes.for: unsupported ptype " + ptype);
108+
}
109+
return MemorySegment.ofArray(bytes);
110+
}
111+
}

‎core/src/main/resources/META-INF/services/io.github.dfa1.vortex.encoding.Decoder‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,4 @@ io.github.dfa1.vortex.encoding.DictCodec
44
io.github.dfa1.vortex.encoding.BitpackedCodec
55
io.github.dfa1.vortex.encoding.DeltaCodec
66
io.github.dfa1.vortex.encoding.SequenceCodec
7+
io.github.dfa1.vortex.encoding.FrameOfReferenceCodec

‎core/src/test/java/io/github/dfa1/vortex/encoding/FrameOfReferenceCodecTest.java‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
package io.github.dfa1.vortex.encoding;
2+
3+
import dev.vortex.proto.ScalarProtos;
4+
import io.github.dfa1.vortex.core.DType;
5+
import io.github.dfa1.vortex.core.PType;
6+
import org.junit.jupiter.api.Test;
7+
import org.junit.jupiter.params.ParameterizedTest;
8+
import org.junit.jupiter.params.provider.ValueSource;
9+
10+
import java.lang.foreign.MemorySegment;
11+
import java.lang.foreign.ValueLayout;
12+
import java.nio.ByteBuffer;
13+
import java.nio.ByteOrder;
14+
15+
import static org.assertj.core.api.Assertions.assertThat;
16+
17+
class FrameOfReferenceCodecTest {
18+
19+
private static final DType I64_DTYPE = new DType.Primitive(PType.I64, false);
20+
private static final DType I32_DTYPE = new DType.Primitive(PType.I32, false);
21+
22+
@Test
23+
void decode_i64_addsReferenceToResiduals() {
24+
// Given
25+
long reference = 1000L;
26+
long[] residuals = {0, 1, 2, 3, 4};
27+
long[] expected = {1000, 1001, 1002, 1003, 1004};
28+
29+
DecodeContext ctx = buildForContext(I64_DTYPE, reference, residuals, PType.I64);
30+
FrameOfReferenceCodec sut = new FrameOfReferenceCodec();
31+
32+
// When
33+
Array result = sut.decode(ctx);
34+
35+
// Then
36+
assertThat(result.length()).isEqualTo(residuals.length);
37+
var layout = ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
38+
for (int i = 0; i < expected.length; i++) {
39+
assertThat(result.buffer(0).get(layout, (long) i * 8))
40+
.as("index %d", i)
41+
.isEqualTo(expected[i]);
42+
}
43+
}
44+
45+
@Test
46+
void decode_i32_addsReferenceToResiduals() {
47+
// Given
48+
long reference = -100L;
49+
long[] residuals = {0, 5, 10, 15};
50+
int[] expected = {-100, -95, -90, -85};
51+
52+
DecodeContext ctx = buildForContext(I32_DTYPE, reference, residuals, PType.I32);
53+
FrameOfReferenceCodec sut = new FrameOfReferenceCodec();
54+
55+
// When
56+
Array result = sut.decode(ctx);
57+
58+
// Then
59+
assertThat(result.length()).isEqualTo(residuals.length);
60+
var layout = ValueLayout.JAVA_INT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
61+
for (int i = 0; i < expected.length; i++) {
62+
assertThat(result.buffer(0).get(layout, (long) i * 4))
63+
.as("index %d", i)
64+
.isEqualTo(expected[i]);
65+
}
66+
}
67+
68+
@Test
69+
void decode_zeroReference_returnsChildUnchanged() {
70+
// Given — reference == 0, should skip the add entirely
71+
long[] residuals = {7, 8, 9};
72+
DecodeContext ctx = buildForContext(I64_DTYPE, 0L, residuals, PType.I64);
73+
FrameOfReferenceCodec sut = new FrameOfReferenceCodec();
74+
75+
// When
76+
Array result = sut.decode(ctx);
77+
78+
// Then — values unchanged
79+
var layout = ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
80+
for (int i = 0; i < residuals.length; i++) {
81+
assertThat(result.buffer(0).get(layout, (long) i * 8)).isEqualTo(residuals[i]);
82+
}
83+
}
84+
85+
@ParameterizedTest
86+
@ValueSource(longs = {Long.MIN_VALUE, Long.MAX_VALUE, -1L, 1L})
87+
void decode_wrappingAdd_i64(long reference) {
88+
// Given — wrapping arithmetic: MAX + 1 wraps to MIN
89+
long[] residuals = {1L};
90+
DecodeContext ctx = buildForContext(I64_DTYPE, reference, residuals, PType.I64);
91+
FrameOfReferenceCodec sut = new FrameOfReferenceCodec();
92+
93+
// When
94+
Array result = sut.decode(ctx);
95+
96+
// Then
97+
var layout = ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
98+
long got = result.buffer(0).get(layout, 0L);
99+
assertThat(got).isEqualTo(residuals[0] + reference);
100+
}
101+
102+
// ── Helpers ───────────────────────────────────────────────────────────────
103+
104+
private static DecodeContext buildForContext(
105+
DType dtype, long reference, long[] residuals, PType ptype
106+
) {
107+
// Serialize the reference as a ScalarValue proto
108+
byte[] metaBytes = ScalarProtos.ScalarValue.newBuilder()
109+
.setInt64Value(reference)
110+
.build()
111+
.toByteArray();
112+
113+
// Build the child primitive buffer
114+
int elemBytes = ptype.byteSize();
115+
byte[] childBytes = new byte[residuals.length * elemBytes];
116+
ByteBuffer bb = ByteBuffer.wrap(childBytes).order(ByteOrder.LITTLE_ENDIAN);
117+
for (long v : residuals) {
118+
switch (ptype) {
119+
case I32, U32 -> bb.putInt((int) v);
120+
case I64, U64 -> bb.putLong(v);
121+
default -> throw new UnsupportedOperationException(ptype.name());
122+
}
123+
}
124+
125+
// ArrayNode for vortex.primitive child: bufferIndices=[0], no children
126+
ArrayNode childNode = new ArrayNode(
127+
"vortex.primitive",
128+
null,
129+
new ArrayNode[0],
130+
new int[]{0},
131+
ArrayStats.empty()
132+
);
133+
134+
// ArrayNode for fastlanes.for: metadata=scalarBytes, child=childNode, no buffers
135+
ArrayNode forNode = new ArrayNode(
136+
"fastlanes.for",
137+
ByteBuffer.wrap(metaBytes),
138+
new ArrayNode[]{childNode},
139+
new int[0],
140+
ArrayStats.empty()
141+
);
142+
143+
MemorySegment[] segments = {MemorySegment.ofArray(childBytes)};
144+
145+
DecoderRegistry registry = DecoderRegistry.empty();
146+
registry.register(new FrameOfReferenceCodec());
147+
registry.register(new PrimitiveCodec());
148+
149+
return new DecodeContext(forNode, dtype, residuals.length, segments, registry);
150+
}
151+
}

0 commit comments

Comments
 (0)