Skip to content

Commit 62fba5a

Browse files
dfa1claude
dfa1
and
claude
committed
feat(encoding): add ALP, Sparse, RunEnd, Constant decoders
Implement the remaining Vortex decoders needed for real-world files written by the Rust JNI writer: - AlpCodec: vortex.alp (F64/F32 → I64/I32 via power-of-10 tables) - SparseCodec: vortex.sparse (fill + patch overlay) - RunEndCodec: vortex.runend (run-length with cumulative end positions) - ConstantCodec: vortex.constant (single ScalarValue broadcast) - DictCodec: add Rust-format path (proto metadata, codes=child[0], values=child[1]) Add proto messages for ALPMetadata, RunEndMetadata, SparseMetadata, PatchesMetadata, BitPackedMetadata, DictMetadata to encodings.proto. Also enable javaReadClose benchmark (previously blocked on missing decoders). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 2005851 commit 62fba5a

16 files changed

Lines changed: 1300 additions & 13 deletions

File tree

‎CLAUDE.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
55
## Commands
66

77
Build prerequisites: `brew install flatbuffers protobuf` (flatc + protoc must be on PATH).
8+
Never use `mvn install` or `./mvwn install`.
89

910
```bash
1011
# Build all modules

‎TODO.md‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444
- Step 2: decode child array recursively via `DecodeContext`
4545
- Step 3: add reference to each element (wrapping add for unsigned types)
4646

47-
- [ ] **#7c Implement `vortex.sparse` decoder**
47+
- [x] **#7c Implement `vortex.sparse` decoder**
4848
- **Spec** (from `encodings/sparse/src/lib.rs`):
4949
- Metadata: protobuf `SparseMetadata` — `patches PatchesMetadata` (tag 1, required)
5050
- Buffer 0: fill value serialized as `ScalarValue` protobuf bytes
@@ -55,7 +55,7 @@
5555
- Step 3: decode patch indices + values from child slots
5656
- Step 4: allocate output, fill with constant, overwrite at patch positions
5757

58-
- [ ] **#7d Implement `vortex.alp` decoder**
58+
- [x] **#7d Implement `vortex.alp` decoder**
5959
- **Spec** (from `encodings/alp/src/alp/array.rs`):
6060
- Metadata: protobuf `ALPMetadata` — `exp_e u32` (tag 1), `exp_f u32` (tag 2), `patches PatchesMetadata` (tag 3, optional)
6161
- Child slot 0: encoded integers (I32 for F32 columns, I64 for F64 columns)
@@ -106,6 +106,19 @@
106106
this is an unrecoverable exception
107107
- drop BufferDesc if not used
108108

109+
## Performance
110+
- in BitpackedCodec, there are a lot of extra allocation like:
111+
try {
112+
byte[] bytes = new byte[rawMeta.remaining()];
113+
rawMeta.duplicate().get(bytes);
114+
meta = EncodingProtos.BitPackedMetadata.parseFrom(bytes);
115+
} catch (InvalidProtocolBufferException e) {
116+
throw new IllegalStateException("fastlanes.bitpacked: invalid metadata", e);
117+
}
118+
=> just use ByteBuffer in this case
119+
- read path should always avoid byte[] allocations or similar
120+
- use MethodHandle to read a long array from a ByteBuffer
121+
109122
## Project
110123
- move the project in a dedicated organization
111124
- create website

‎core/src/main/java/io/github/dfa1/vortex/encoding/AlpCodec.java‎

Lines changed: 221 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
package io.github.dfa1.vortex.encoding;
2+
3+
import com.google.protobuf.InvalidProtocolBufferException;
4+
import dev.vortex.proto.DTypeProtos;
5+
import dev.vortex.proto.EncodingProtos;
6+
import io.github.dfa1.vortex.core.DType;
7+
import io.github.dfa1.vortex.core.PType;
8+
9+
import java.lang.foreign.MemorySegment;
10+
import java.lang.foreign.ValueLayout;
11+
import java.nio.ByteBuffer;
12+
import java.nio.ByteOrder;
13+
14+
/// Decoder for {@code vortex.alp} — Adaptive Lossless floating-Point compression.
15+
///
16+
/// <p>Metadata: protobuf {@code ALPMetadata} — {@code exp_e u32} (tag 1),
17+
/// {@code exp_f u32} (tag 2), optional {@code patches PatchesMetadata} (tag 3).
18+
///
19+
/// <p>Child slot 0: encoded integers (I32 for F32, I64 for F64 parent), rowCount = array len.
20+
/// Child slot 1: patch indices (if patches), rowCount = patches.len.
21+
/// Child slot 2: patch values (if patches, same dtype as parent), rowCount = patches.len.
22+
/// Child slot 3: chunk offsets (optional, ignored — patches applied by absolute index).
23+
///
24+
/// <p>Decode: {@code decoded[i] = (float/double) encoded[i] * F10[exp_f] * IF10[exp_e]},
25+
/// then overwrite {@code decoded[indices[j] - offset] = values[j]} for each patch.
26+
public final class AlpCodec implements Decoder {
27+
28+
// Powers of 10 for F64 (index 0..18 used by the encoder).
29+
private static final double[] F10_F64 = {
30+
1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9,
31+
1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19,
32+
1e20, 1e21, 1e22, 1e23
33+
};
34+
35+
private static final double[] IF10_F64 = {
36+
1e-0, 1e-1, 1e-2, 1e-3, 1e-4, 1e-5, 1e-6, 1e-7, 1e-8, 1e-9,
37+
1e-10, 1e-11, 1e-12, 1e-13, 1e-14, 1e-15, 1e-16, 1e-17, 1e-18, 1e-19,
38+
1e-20, 1e-21, 1e-22, 1e-23
39+
};
40+
41+
// Powers of 10 for F32.
42+
private static final float[] F10_F32 = {
43+
1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, 1e6f, 1e7f, 1e8f, 1e9f, 1e10f
44+
};
45+
46+
private static final float[] IF10_F32 = {
47+
1e-0f, 1e-1f, 1e-2f, 1e-3f, 1e-4f, 1e-5f, 1e-6f, 1e-7f, 1e-8f, 1e-9f, 1e-10f
48+
};
49+
50+
@Override
51+
public String encodingId() {
52+
return "vortex.alp";
53+
}
54+
55+
@Override
56+
public Array decode(DecodeContext ctx) {
57+
ByteBuffer rawMeta = ctx.metadata();
58+
if (rawMeta == null) {
59+
throw new IllegalStateException("vortex.alp: missing metadata");
60+
}
61+
byte[] metaBytes = new byte[rawMeta.remaining()];
62+
rawMeta.duplicate().get(metaBytes);
63+
64+
EncodingProtos.ALPMetadata meta;
65+
try {
66+
meta = EncodingProtos.ALPMetadata.parseFrom(metaBytes);
67+
} catch (InvalidProtocolBufferException e) {
68+
throw new IllegalStateException("vortex.alp: invalid metadata", e);
69+
}
70+
71+
if (!(ctx.dtype() instanceof DType.Primitive p)) {
72+
throw new IllegalStateException("vortex.alp: expected primitive dtype, got " + ctx.dtype());
73+
}
74+
75+
int expE = meta.getExpE();
76+
int expF = meta.getExpF();
77+
PType ptype = p.ptype();
78+
long n = ctx.rowCount();
79+
80+
return switch (ptype) {
81+
case F64 -> decodeF64(ctx, meta, expE, expF, n);
82+
case F32 -> decodeF32(ctx, meta, expE, expF, n);
83+
default -> throw new IllegalStateException("vortex.alp: unsupported dtype " + ptype);
84+
};
85+
}
86+
87+
// ── F64 ───────────────────────────────────────────────────────────────────
88+
89+
private Array decodeF64(DecodeContext ctx, EncodingProtos.ALPMetadata meta, int expE, int expF, long n) {
90+
DType encodedDtype = new DType.Primitive(PType.I64, false);
91+
Array encoded = decodeChildAs(ctx, 0, encodedDtype, n);
92+
93+
double f10 = F10_F64[expF];
94+
double if10 = IF10_F64[expE];
95+
96+
byte[] outBytes = new byte[(int) (n * 8)];
97+
ByteBuffer out = ByteBuffer.wrap(outBytes).order(ByteOrder.LITTLE_ENDIAN);
98+
var srcLayout = ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
99+
MemorySegment src = encoded.buffer(0);
100+
for (long i = 0; i < n; i++) {
101+
long encVal = src.get(srcLayout, i * 8);
102+
double dec = (double) encVal * f10 * if10;
103+
out.putDouble(dec);
104+
}
105+
106+
if (meta.hasPatches()) {
107+
applyPatchesF64(ctx, meta.getPatches(), outBytes, n);
108+
}
109+
110+
return new Array(ctx.dtype(), n,
111+
new MemorySegment[]{MemorySegment.ofArray(outBytes)}, new Array[0], ArrayStats.empty());
112+
}
113+
114+
private void applyPatchesF64(DecodeContext ctx, EncodingProtos.PatchesMetadata pm,
115+
byte[] outBytes, long n) {
116+
long numPatches = pm.getLen();
117+
long offset = pm.getOffset();
118+
PType idxPtype = ptypeFromProto(pm.getIndicesPtype());
119+
120+
DType idxDtype = new DType.Primitive(idxPtype, false);
121+
DType valDtype = ctx.dtype();
122+
123+
Array idxArr = decodeChildAs(ctx, 1, idxDtype, numPatches);
124+
Array valArr = decodeChildAs(ctx, 2, valDtype, numPatches);
125+
126+
var idxLayout = unsignedLayout(idxPtype);
127+
var valLayout = ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
128+
129+
ByteBuffer out = ByteBuffer.wrap(outBytes).order(ByteOrder.LITTLE_ENDIAN);
130+
MemorySegment idxSeg = idxArr.buffer(0);
131+
MemorySegment valSeg = valArr.buffer(0);
132+
133+
for (long i = 0; i < numPatches; i++) {
134+
long absIdx = readUnsigned(idxSeg, i, idxPtype) - offset;
135+
long rawBits = valSeg.get(valLayout, i * 8);
136+
out.putLong((int) (absIdx * 8), rawBits);
137+
}
138+
}
139+
140+
// ── F32 ───────────────────────────────────────────────────────────────────
141+
142+
private Array decodeF32(DecodeContext ctx, EncodingProtos.ALPMetadata meta, int expE, int expF, long n) {
143+
DType encodedDtype = new DType.Primitive(PType.I32, false);
144+
Array encoded = decodeChildAs(ctx, 0, encodedDtype, n);
145+
146+
float f10 = F10_F32[expF];
147+
float if10 = IF10_F32[expE];
148+
149+
byte[] outBytes = new byte[(int) (n * 4)];
150+
ByteBuffer out = ByteBuffer.wrap(outBytes).order(ByteOrder.LITTLE_ENDIAN);
151+
var srcLayout = ValueLayout.JAVA_INT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
152+
MemorySegment src = encoded.buffer(0);
153+
for (long i = 0; i < n; i++) {
154+
int encVal = src.get(srcLayout, i * 4);
155+
float dec = (float) encVal * f10 * if10;
156+
out.putFloat(dec);
157+
}
158+
159+
if (meta.hasPatches()) {
160+
applyPatchesF32(ctx, meta.getPatches(), outBytes, n);
161+
}
162+
163+
return new Array(ctx.dtype(), n,
164+
new MemorySegment[]{MemorySegment.ofArray(outBytes)}, new Array[0], ArrayStats.empty());
165+
}
166+
167+
private void applyPatchesF32(DecodeContext ctx, EncodingProtos.PatchesMetadata pm,
168+
byte[] outBytes, long n) {
169+
long numPatches = pm.getLen();
170+
long offset = pm.getOffset();
171+
PType idxPtype = ptypeFromProto(pm.getIndicesPtype());
172+
173+
DType idxDtype = new DType.Primitive(idxPtype, false);
174+
DType valDtype = ctx.dtype();
175+
176+
Array idxArr = decodeChildAs(ctx, 1, idxDtype, numPatches);
177+
Array valArr = decodeChildAs(ctx, 2, valDtype, numPatches);
178+
179+
var valLayout = ValueLayout.JAVA_INT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
180+
181+
ByteBuffer out = ByteBuffer.wrap(outBytes).order(ByteOrder.LITTLE_ENDIAN);
182+
MemorySegment idxSeg = idxArr.buffer(0);
183+
MemorySegment valSeg = valArr.buffer(0);
184+
185+
for (long i = 0; i < numPatches; i++) {
186+
long absIdx = readUnsigned(idxSeg, i, idxPtype) - offset;
187+
int rawBits = valSeg.get(valLayout, i * 4);
188+
out.putInt((int) (absIdx * 4), rawBits);
189+
}
190+
}
191+
192+
// ── Helpers ───────────────────────────────────────────────────────────────
193+
194+
private static Array decodeChildAs(DecodeContext parent, int childIdx, DType dtype, long rowCount) {
195+
ArrayNode childNode = parent.node().children()[childIdx];
196+
DecodeContext childCtx = new DecodeContext(
197+
childNode, dtype, rowCount, parent.segmentBuffers(), parent.registry());
198+
return parent.registry().decode(childCtx);
199+
}
200+
201+
private static long readUnsigned(MemorySegment seg, long i, PType ptype) {
202+
return switch (ptype) {
203+
case U8 -> Byte.toUnsignedLong(seg.get(ValueLayout.JAVA_BYTE, i));
204+
case U16 -> Short.toUnsignedLong(
205+
seg.get(ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN), i * 2));
206+
case U32 -> Integer.toUnsignedLong(
207+
seg.get(ValueLayout.JAVA_INT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN), i * 4));
208+
case U64 -> seg.get(
209+
ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN), i * 8);
210+
default -> throw new IllegalStateException("vortex.alp: non-unsigned patch index ptype " + ptype);
211+
};
212+
}
213+
214+
private static ValueLayout.OfLong unsignedLayout(PType ptype) {
215+
return ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
216+
}
217+
218+
private static PType ptypeFromProto(DTypeProtos.PType proto) {
219+
return PType.values()[proto.getNumber()];
220+
}
221+
}

‎core/src/main/java/io/github/dfa1/vortex/encoding/BufferDesc.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package io.github.dfa1.vortex.encoding;
22

33
/// Describes one data buffer within an array segment.
4+
/// TODO: unused, remove?
45
public record BufferDesc(
56
short padding,
67
byte alignmentExponent,

‎core/src/main/java/io/github/dfa1/vortex/encoding/ConstantCodec.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
package io.github.dfa1.vortex.encoding;
2+
3+
import com.google.protobuf.InvalidProtocolBufferException;
4+
import dev.vortex.proto.ScalarProtos;
5+
import io.github.dfa1.vortex.core.DType;
6+
import io.github.dfa1.vortex.core.PType;
7+
8+
import java.lang.foreign.MemorySegment;
9+
import java.lang.foreign.ValueLayout;
10+
import java.nio.ByteBuffer;
11+
import java.nio.ByteOrder;
12+
13+
/// Decoder for {@code vortex.constant} — all elements share the same value.
14+
///
15+
/// <p>No metadata (empty bytes). Buffer 0: the constant value as raw {@code ScalarValue}
16+
/// proto bytes. No children.
17+
///
18+
/// <p>Decode: fill an output buffer of {@code rowCount} elements with the constant value.
19+
public final class ConstantCodec implements Decoder {
20+
21+
@Override
22+
public String encodingId() {
23+
return "vortex.constant";
24+
}
25+
26+
@Override
27+
public Array decode(DecodeContext ctx) {
28+
if (!(ctx.dtype() instanceof DType.Primitive p)) {
29+
throw new IllegalStateException("vortex.constant: expected primitive dtype, got " + ctx.dtype());
30+
}
31+
32+
MemorySegment scalarBuf = ctx.buffer(0);
33+
byte[] scalarBytes = scalarBuf.toArray(ValueLayout.JAVA_BYTE);
34+
35+
ScalarProtos.ScalarValue scalar;
36+
try {
37+
scalar = ScalarProtos.ScalarValue.parseFrom(scalarBytes);
38+
} catch (InvalidProtocolBufferException e) {
39+
throw new IllegalStateException("vortex.constant: invalid scalar value", e);
40+
}
41+
42+
PType ptype = p.ptype();
43+
long n = ctx.rowCount();
44+
int elemBytes = ptype.byteSize();
45+
long rawBits = scalarToRawBits(scalar, ptype);
46+
47+
byte[] outBytes = new byte[(int) (n * elemBytes)];
48+
ByteBuffer out = ByteBuffer.wrap(outBytes).order(ByteOrder.LITTLE_ENDIAN);
49+
for (long i = 0; i < n; i++) {
50+
writeRaw(out, ptype, rawBits);
51+
}
52+
53+
return new Array(ctx.dtype(), n,
54+
new MemorySegment[]{MemorySegment.ofArray(outBytes)}, new Array[0], ArrayStats.empty());
55+
}
56+
57+
// ── Helpers ───────────────────────────────────────────────────────────────
58+
59+
private static long scalarToRawBits(ScalarProtos.ScalarValue scalar, PType ptype) {
60+
return switch (scalar.getKindCase()) {
61+
case INT64_VALUE -> scalar.getInt64Value();
62+
case UINT64_VALUE -> scalar.getUint64Value();
63+
case F32_VALUE -> Float.floatToRawIntBits(scalar.getF32Value());
64+
case F64_VALUE -> Double.doubleToRawLongBits(scalar.getF64Value());
65+
case KIND_NOT_SET -> 0L;
66+
default -> throw new IllegalStateException(
67+
"vortex.constant: unexpected scalar kind " + scalar.getKindCase());
68+
};
69+
}
70+
71+
private static void writeRaw(ByteBuffer buf, PType ptype, long rawBits) {
72+
switch (ptype.byteSize()) {
73+
case 1 -> buf.put((byte) rawBits);
74+
case 2 -> buf.putShort((short) rawBits);
75+
case 4 -> buf.putInt((int) rawBits);
76+
case 8 -> buf.putLong(rawBits);
77+
default -> throw new UnsupportedOperationException("vortex.constant: unsupported ptype " + ptype);
78+
}
79+
}
80+
}

‎core/src/main/java/io/github/dfa1/vortex/encoding/Decoder.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,10 @@
11
package io.github.dfa1.vortex.encoding;
22

3-
import java.io.IOException;
4-
53
/// Decodes one encoding type into a flat [Array].
64
///
75
/// Register implementations via `ServiceLoader` (META-INF/services) or
86
/// [DecoderRegistry#register(Decoder)].
7+
/// TODO: merge inside Codec
98
public interface Decoder {
109
/// Encoding ID this decoder handles, e.g. `"fastlanes.bitpacked"`.
1110
String encodingId();

‎core/src/main/java/io/github/dfa1/vortex/encoding/DecoderRegistry.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package io.github.dfa1.vortex.encoding;
22

3-
import java.io.IOException;
43
import java.util.HashMap;
54
import java.util.Map;
65
import java.util.ServiceLoader;

0 commit comments

Comments
 (0)