Skip to content

Commit 937ade3

Browse files
dfa1claude
dfa1
and
claude
committed
feat(reader): lazy ALP-RD decode via top-level records
Replace the eager ALP-RD expansion with two top-level record types in reader.array: LazyAlpRdDoubleArray and LazyAlpRdFloatArray. ALP-RD splits each f64/f32 into a u16 "left" dictionary code and a u64 (f64) or u32 (f32) "right" payload (the low rightBitWidth bits of the original IEEE-754 bit pattern). Reconstruction is: bits = (dict[left[i]] << rightBitWidth) | right[i] result = longBitsToDouble(bits) // or intBitsToFloat for f32 Optional patches override the dict lookup at specific absolute indices for outlier values that don't compress through the dictionary. getDouble(i) / getFloat(i) read left + right children, dispatch through dict (or patch via binary search on patchIndices when present), shift+or, then bits-to-double/float. No n-sized output buffer allocated. The decoder pulls the small (typically ≤ 16-entry) dict from metadata into a short[] as before; left + right children are kept as typed ShortArray + LongArray/IntArray (no full materialise); patches collapse to (patchIndices: Array, patchLeftValues: short[], offset: long). The patchIndices Array is whichever narrow-int type the encoder chose (U8/U16/U32/U64) — looked up via SparseArrays.findPatch which already centralises the unsigned-index binary search. Bool and VarBin are not in scope for ALP-RD; accepts() restricts to F32/F64. AlpRdEncodingEncoderTest converted to use typed DoubleArray.getDouble / FloatArray.getFloat instead of ArraySegments.of(_).get(LE_xx, off), which no longer works on lazy outputs. 3 new unit tests in LazyAlpRdArrayTest cover f64 dict reconstruction, f32 dict reconstruction, and the patches override path. ./mvnw verify green (13 modules, integration suite 48s). Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
1 parent d83ec1b commit 937ade3

5 files changed

Lines changed: 309 additions & 87 deletions

File tree

‎reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyAlpRdDoubleArray.java‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
package io.github.dfa1.vortex.reader.array;
2+
3+
import io.github.dfa1.vortex.core.DType;
4+
5+
import java.util.function.DoubleBinaryOperator;
6+
import java.util.function.DoubleConsumer;
7+
8+
/// Lazy ALP-RD-encoded {@link DoubleArray}.
9+
///
10+
/// ALP-RD splits each f64 into a "left" u16 dictionary code and a "right" u64 payload
11+
/// (the low `rightBitWidth` bits of the original bit pattern). Reconstruction is
12+
/// `Double.longBitsToDouble((dict[left[i]] << rightBitWidth) | right[i])`. A small
13+
/// patches table overrides the dict lookup at specific absolute indices for outlier
14+
/// values that don't compress through the dictionary.
15+
///
16+
/// {@code getDouble(i)} resolves on demand: read left/right children, dispatch through
17+
/// dict (or patch if `i + patchOffset` hits a patch index), shift+or, then bits-to-double.
18+
///
19+
/// @param dtype logical element type
20+
/// @param length total logical row count
21+
/// @param dict small u16 dictionary lookup table (length typically ≤ 16)
22+
/// @param rightBitWidth bit width of the right payload
23+
/// @param leftArr per-row u16 dict codes
24+
/// @param rightArr per-row u64 right payloads (raw bits)
25+
/// @param patchIndices sorted absolute indices of patched rows (or {@code null} when no patches)
26+
/// @param patchLeftValues actual left u16 values for patched rows; aligned with
27+
/// {@code patchIndices}
28+
/// @param patchOffset absolute origin subtracted from row positions before patch lookup
29+
public record LazyAlpRdDoubleArray(
30+
DType dtype, long length,
31+
short[] dict, int rightBitWidth,
32+
ShortArray leftArr, LongArray rightArr,
33+
Array patchIndices, short[] patchLeftValues, long patchOffset)
34+
implements DoubleArray {
35+
36+
@Override
37+
public double getDouble(long i) {
38+
int leftU16 = lookupLeft(i);
39+
long leftBits = ((long) (leftU16 & 0xFFFF)) << rightBitWidth;
40+
long rightBits = rightArr.getLong(i);
41+
return Double.longBitsToDouble(leftBits | rightBits);
42+
}
43+
44+
@Override
45+
public void forEachDouble(DoubleConsumer c) {
46+
long n = length;
47+
for (long i = 0; i < n; i++) {
48+
c.accept(getDouble(i));
49+
}
50+
}
51+
52+
@Override
53+
public double fold(double identity, DoubleBinaryOperator op) {
54+
double acc = identity;
55+
long n = length;
56+
for (long i = 0; i < n; i++) {
57+
acc = op.applyAsDouble(acc, getDouble(i));
58+
}
59+
return acc;
60+
}
61+
62+
private int lookupLeft(long i) {
63+
if (patchIndices != null) {
64+
int p = SparseArrays.findPatch(patchIndices, patchLeftValues.length, i + patchOffset);
65+
if (p >= 0) {
66+
return patchLeftValues[p];
67+
}
68+
}
69+
short code = leftArr.getShort(i);
70+
return dict[Short.toUnsignedInt(code)];
71+
}
72+
}

‎reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyAlpRdFloatArray.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
package io.github.dfa1.vortex.reader.array;
2+
3+
import io.github.dfa1.vortex.core.DType;
4+
5+
import java.util.function.DoubleBinaryOperator;
6+
7+
/// Lazy ALP-RD-encoded {@link FloatArray}. See {@link LazyAlpRdDoubleArray} for the
8+
/// dict + right + patches reconstruction model. The right payload is u32 instead of
9+
/// u64 and the result is reinterpreted via {@link Float#intBitsToFloat}.
10+
///
11+
/// @param dtype logical element type
12+
/// @param length total logical row count
13+
/// @param dict small u16 dictionary lookup table
14+
/// @param rightBitWidth bit width of the right payload
15+
/// @param leftArr per-row u16 dict codes
16+
/// @param rightArr per-row u32 right payloads (raw bits)
17+
/// @param patchIndices sorted absolute indices of patched rows (or {@code null})
18+
/// @param patchLeftValues actual left u16 values for patched rows
19+
/// @param patchOffset absolute origin subtracted from row positions before patch lookup
20+
public record LazyAlpRdFloatArray(
21+
DType dtype, long length,
22+
short[] dict, int rightBitWidth,
23+
ShortArray leftArr, IntArray rightArr,
24+
Array patchIndices, short[] patchLeftValues, long patchOffset)
25+
implements FloatArray {
26+
27+
@Override
28+
public float getFloat(long i) {
29+
int leftU16 = lookupLeft(i);
30+
int leftBits = (leftU16 & 0xFFFF) << rightBitWidth;
31+
int rightBits = rightArr.getInt(i);
32+
return Float.intBitsToFloat(leftBits | rightBits);
33+
}
34+
35+
@Override
36+
public double fold(double identity, DoubleBinaryOperator op) {
37+
double acc = identity;
38+
long n = length;
39+
for (long i = 0; i < n; i++) {
40+
acc = op.applyAsDouble(acc, getFloat(i));
41+
}
42+
return acc;
43+
}
44+
45+
private int lookupLeft(long i) {
46+
if (patchIndices != null) {
47+
int p = SparseArrays.findPatch(patchIndices, patchLeftValues.length, i + patchOffset);
48+
if (p >= 0) {
49+
return patchLeftValues[p];
50+
}
51+
}
52+
short code = leftArr.getShort(i);
53+
return dict[Short.toUnsignedInt(code)];
54+
}
55+
}

‎reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java‎

Lines changed: 44 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,15 @@
88
import io.github.dfa1.vortex.proto.ALPRDMetadata;
99
import io.github.dfa1.vortex.proto.PatchesMetadata;
1010
import io.github.dfa1.vortex.reader.array.Array;
11-
import io.github.dfa1.vortex.reader.array.MaterializedDoubleArray;
12-
import io.github.dfa1.vortex.reader.array.MaterializedFloatArray;
11+
import io.github.dfa1.vortex.reader.array.IntArray;
12+
import io.github.dfa1.vortex.reader.array.LazyAlpRdDoubleArray;
13+
import io.github.dfa1.vortex.reader.array.LazyAlpRdFloatArray;
14+
import io.github.dfa1.vortex.reader.array.LongArray;
15+
import io.github.dfa1.vortex.reader.array.MaskedArray;
16+
import io.github.dfa1.vortex.reader.array.ShortArray;
1317

1418
import java.io.IOException;
1519
import java.lang.foreign.MemorySegment;
16-
import java.lang.foreign.ValueLayout;
1720
import java.nio.ByteBuffer;
1821

1922
/// Read-only decoder for {@code vortex.alprd}.
@@ -58,104 +61,61 @@ public Array decode(DecodeContext ctx) {
5861
long n = ctx.rowCount();
5962
PType ptype = p.ptype();
6063

64+
// Lazy path: keep left/right as typed Arrays + patches as a small short[] +
65+
// a lazy indices Array. No n-sized output buffer allocated.
66+
Array leftRaw = ctx.decodeChild(0, U16_DTYPE, n);
67+
ShortArray leftArr = (ShortArray) unwrap(leftRaw);
68+
69+
Patches patches = decodePatches(ctx, meta.patches());
70+
6171
return switch (ptype) {
62-
case F64 -> decodeF64(ctx, meta, dict, rightBitWidth, n);
63-
case F32 -> decodeF32(ctx, meta, dict, rightBitWidth, n);
72+
case F64 -> {
73+
Array rightRaw = ctx.decodeChild(1, U64_DTYPE, n);
74+
LongArray rightArr = (LongArray) unwrap(rightRaw);
75+
yield new LazyAlpRdDoubleArray(ctx.dtype(), n, dict, rightBitWidth,
76+
leftArr, rightArr, patches.indices, patches.leftValues, patches.offset);
77+
}
78+
case F32 -> {
79+
Array rightRaw = ctx.decodeChild(1, U32_DTYPE, n);
80+
IntArray rightArr = (IntArray) unwrap(rightRaw);
81+
yield new LazyAlpRdFloatArray(ctx.dtype(), n, dict, rightBitWidth,
82+
leftArr, rightArr, patches.indices, patches.leftValues, patches.offset);
83+
}
6484
default -> throw new VortexException(EncodingId.VORTEX_ALPRD, "unsupported dtype " + ptype);
6585
};
6686
}
6787

68-
private static Array decodeF64(DecodeContext ctx, ALPRDMetadata meta, short[] dict, int rightBitWidth, long n) {
69-
MemorySegment leftSeg = ctx.decodeChildSegment(0, U16_DTYPE, n);
70-
MemorySegment rightSeg = ctx.decodeChildSegment(1, U64_DTYPE, n);
71-
long leftCap = SegmentBroadcast.capacity(leftSeg, 2);
72-
long rightCap = SegmentBroadcast.capacity(rightSeg, 8);
73-
MemorySegment out = ctx.arena().allocate(n * Long.BYTES, Long.BYTES);
74-
75-
for (long i = 0; i < n; i++) {
76-
int code = Short.toUnsignedInt(leftSeg.getAtIndex(PTypeIO.LE_SHORT, i % leftCap));
77-
long leftBits = (long) (dict[code] & 0xFFFF) << rightBitWidth;
78-
long rightBits = rightSeg.getAtIndex(PTypeIO.LE_LONG, i % rightCap);
79-
out.setAtIndex(PTypeIO.LE_LONG, i, leftBits | rightBits);
80-
}
81-
82-
if (meta.patches() != null) {
83-
applyPatchesF64(ctx, meta.patches(), out, rightSeg, rightCap, rightBitWidth);
84-
}
85-
86-
return new MaterializedDoubleArray(ctx.dtype(), n, out.asReadOnly());
88+
private static Array unwrap(Array arr) {
89+
return arr instanceof MaskedArray m ? m.inner() : arr;
8790
}
8891

89-
private static Array decodeF32(DecodeContext ctx, ALPRDMetadata meta, short[] dict, int rightBitWidth, long n) {
90-
MemorySegment leftSeg = ctx.decodeChildSegment(0, U16_DTYPE, n);
91-
MemorySegment rightSeg = ctx.decodeChildSegment(1, U32_DTYPE, n);
92-
long leftCap = SegmentBroadcast.capacity(leftSeg, 2);
93-
long rightCap = SegmentBroadcast.capacity(rightSeg, 4);
94-
MemorySegment out = ctx.arena().allocate(n * Integer.BYTES, Integer.BYTES);
95-
96-
for (long i = 0; i < n; i++) {
97-
int code = Short.toUnsignedInt(leftSeg.getAtIndex(PTypeIO.LE_SHORT, i % leftCap));
98-
int leftBits = (dict[code] & 0xFFFF) << rightBitWidth;
99-
int rightBits = rightSeg.getAtIndex(PTypeIO.LE_INT, i % rightCap);
100-
out.setAtIndex(PTypeIO.LE_INT, i, leftBits | rightBits);
101-
}
102-
103-
if (meta.patches() != null) {
104-
applyPatchesF32(ctx, meta.patches(), out, rightSeg, rightCap, rightBitWidth);
105-
}
106-
107-
return new MaterializedFloatArray(ctx.dtype(), n, out.asReadOnly());
92+
/// Decoded patches: sorted absolute indices (as a typed Array for in-place lookup)
93+
/// plus the actual left u16 values pulled into a short[].
94+
private record Patches(Array indices, short[] leftValues, long offset) {
95+
static final Patches EMPTY = new Patches(null, new short[0], 0L);
10896
}
10997

110-
private static void applyPatchesF64(DecodeContext ctx, PatchesMetadata pm,
111-
MemorySegment out, MemorySegment rightSeg, long rightCap, int rightBitWidth) {
112-
long numPatches = pm.len();
113-
long offset = pm.offset();
114-
PType idxPtype = PType.fromOrdinal(pm.indices_ptype().value());
115-
116-
MemorySegment idxSeg = ctx.decodeChildSegment(2, new DType.Primitive(idxPtype, false), numPatches);
117-
MemorySegment valSeg = ctx.decodeChildSegment(3, U16_DTYPE, numPatches);
118-
int idxBytes = idxPtype.byteSize();
119-
long valCap = SegmentBroadcast.capacity(valSeg, 2);
120-
121-
for (long j = 0; j < numPatches; j++) {
122-
long absIdx = readUnsigned(idxSeg, SegmentBroadcast.elementOffset(idxSeg, j, idxBytes), idxPtype) - offset;
123-
short actualLeftU16 = valSeg.getAtIndex(PTypeIO.LE_SHORT, j % valCap);
124-
long leftBits = (long) (actualLeftU16 & 0xFFFF) << rightBitWidth;
125-
long rightBits = rightSeg.getAtIndex(PTypeIO.LE_LONG, absIdx % rightCap);
126-
out.setAtIndex(PTypeIO.LE_LONG, absIdx, leftBits | rightBits);
98+
private static Patches decodePatches(DecodeContext ctx, PatchesMetadata pm) {
99+
if (pm == null || pm.len() == 0) {
100+
return Patches.EMPTY;
127101
}
128-
}
129-
130-
private static void applyPatchesF32(DecodeContext ctx, PatchesMetadata pm,
131-
MemorySegment out, MemorySegment rightSeg, long rightCap, int rightBitWidth) {
132102
long numPatches = pm.len();
133103
long offset = pm.offset();
134104
PType idxPtype = PType.fromOrdinal(pm.indices_ptype().value());
105+
DType idxDtype = new DType.Primitive(idxPtype, false);
135106

136-
MemorySegment idxSeg = ctx.decodeChildSegment(2, new DType.Primitive(idxPtype, false), numPatches);
107+
Array idxArr = ctx.decodeChild(2, idxDtype, numPatches);
108+
Array idxData = idxArr instanceof MaskedArray m ? m.inner() : idxArr;
109+
110+
// Pull the small left-values table into a short[] so lookups don't pay an
111+
// Array-dispatch per patch hit. Patches are typically <1% of rows.
137112
MemorySegment valSeg = ctx.decodeChildSegment(3, U16_DTYPE, numPatches);
138-
int idxBytes = idxPtype.byteSize();
139113
long valCap = SegmentBroadcast.capacity(valSeg, 2);
140-
141-
for (long j = 0; j < numPatches; j++) {
142-
long absIdx = readUnsigned(idxSeg, SegmentBroadcast.elementOffset(idxSeg, j, idxBytes), idxPtype) - offset;
143-
short actualLeftU16 = valSeg.getAtIndex(PTypeIO.LE_SHORT, j % valCap);
144-
int leftBits = (actualLeftU16 & 0xFFFF) << rightBitWidth;
145-
int rightBits = rightSeg.getAtIndex(PTypeIO.LE_INT, absIdx % rightCap);
146-
out.setAtIndex(PTypeIO.LE_INT, (int) absIdx, leftBits | rightBits);
114+
short[] leftValues = new short[(int) numPatches];
115+
for (int j = 0; j < numPatches; j++) {
116+
leftValues[j] = valSeg.getAtIndex(PTypeIO.LE_SHORT, j % valCap);
147117
}
148-
}
149-
150-
private static long readUnsigned(MemorySegment seg, long off, PType ptype) {
151-
return switch (ptype) {
152-
case U8 -> Byte.toUnsignedLong(seg.get(ValueLayout.JAVA_BYTE, off));
153-
case U16 -> Short.toUnsignedLong(seg.get(PTypeIO.LE_SHORT, off));
154-
case U32 -> Integer.toUnsignedLong(seg.get(PTypeIO.LE_INT, off));
155-
case U64 -> seg.get(PTypeIO.LE_LONG, off);
156-
default -> throw new VortexException(EncodingId.VORTEX_ALPRD,
157-
"non-unsigned patch index ptype " + ptype);
158-
};
118+
return new Patches(idxData, leftValues, offset);
159119
}
160120

161121
private static ALPRDMetadata parseMeta(DecodeContext ctx) {

0 commit comments

Comments
 (0)