Skip to content

Commit 3082745

Browse files
dfa1claude
dfa1
and
claude
committed
fix(sparse): handle NULL_VALUE fill and non-primitive dtypes
Three decoder bugs found while reading sparse.vortex S3 fixture: - NULL_VALUE scalar fill threw instead of mapping to 0 - Utf8/Binary dtype threw "expected primitive dtype" - Bool dtype threw "expected primitive dtype" Added decodeVarBin and decodeBool paths, plus regression tests for each failure mode. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent f1e1ea0 commit 3082745

2 files changed

Lines changed: 253 additions & 2 deletions

File tree

‎core/src/main/java/io/github/dfa1/vortex/encoding/SparseEncoding.java‎

Lines changed: 96 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,14 @@
88
import io.github.dfa1.vortex.core.ArrayStats;
99
import io.github.dfa1.vortex.core.DType;
1010
import io.github.dfa1.vortex.core.PType;
11+
import io.github.dfa1.vortex.core.array.BoolArray;
1112
import io.github.dfa1.vortex.core.array.ByteArray;
1213
import io.github.dfa1.vortex.core.array.DoubleArray;
1314
import io.github.dfa1.vortex.core.array.FloatArray;
1415
import io.github.dfa1.vortex.core.array.IntArray;
1516
import io.github.dfa1.vortex.core.array.LongArray;
1617
import io.github.dfa1.vortex.core.array.ShortArray;
18+
import io.github.dfa1.vortex.core.array.VarBinArray;
1719
import io.github.dfa1.vortex.core.VortexException;
1820

1921
import java.lang.foreign.Arena;
@@ -184,6 +186,16 @@ private static Array decode(DecodeContext ctx) {
184186
long offset = patches.getOffset();
185187
PType indicesPtype = ptypeFromProto(patches.getIndicesPtype());
186188

189+
long n = ctx.rowCount();
190+
191+
if (ctx.dtype() instanceof DType.Utf8 || ctx.dtype() instanceof DType.Binary) {
192+
return decodeVarBin(ctx, n, numPatches, offset, indicesPtype);
193+
}
194+
195+
if (ctx.dtype() instanceof DType.Bool) {
196+
return decodeBool(ctx, n, numPatches, offset, indicesPtype);
197+
}
198+
187199
if (!(ctx.dtype() instanceof DType.Primitive)) {
188200
throw new VortexException(EncodingId.VORTEX_SPARSE, "expected primitive dtype, got " + ctx.dtype());
189201
}
@@ -197,7 +209,6 @@ private static Array decode(DecodeContext ctx) {
197209
throw new VortexException(EncodingId.VORTEX_SPARSE, "invalid fill value", e);
198210
}
199211

200-
long n = ctx.rowCount();
201212
int elemBytes = valuePtype.byteSize();
202213
MemorySegment out = ctx.arena().allocate(n * elemBytes);
203214
fillSegment(out, n, valuePtype, fillScalar);
@@ -221,6 +232,89 @@ private static Array decode(DecodeContext ctx) {
221232
};
222233
}
223234

235+
private static Array decodeBool(
236+
DecodeContext ctx, long n, long numPatches, long offset, PType indicesPtype
237+
) {
238+
long numBytes = (n + 7) >>> 3;
239+
MemorySegment out = ctx.arena().allocate(numBytes);
240+
if (numPatches > 0) {
241+
DType indicesDtype = new DType.Primitive(indicesPtype, false);
242+
Array indicesArray = decodeChild(ctx, 0, indicesDtype, numPatches);
243+
Array valuesArray = decodeChild(ctx, 1, ctx.dtype(), numPatches);
244+
MemorySegment idxSeg = indicesArray.buffer(0);
245+
BoolArray bools = (BoolArray) valuesArray;
246+
for (long i = 0; i < numPatches; i++) {
247+
if (bools.getBoolean(i)) {
248+
long pos = readUnsignedIdx(idxSeg, i, indicesPtype) - offset;
249+
long byteIdx = pos >>> 3;
250+
byte cur = out.get(ValueLayout.JAVA_BYTE, byteIdx);
251+
out.set(ValueLayout.JAVA_BYTE, byteIdx, (byte) (cur | (1 << (pos & 7))));
252+
}
253+
}
254+
}
255+
return new BoolArray(ctx.dtype(), n, out, ArrayStats.empty());
256+
}
257+
258+
private static Array decodeVarBin(
259+
DecodeContext ctx, long n, long numPatches, long offset, PType indicesPtype
260+
) {
261+
MemorySegment outOffsets = ctx.arena().allocate((n + 1) * 4L, 4);
262+
if (numPatches == 0) {
263+
MemorySegment outBytes = ctx.arena().allocate(1);
264+
DType i32dtype = new DType.Primitive(PType.I32, false);
265+
Array offsetArr = new IntArray(i32dtype, n + 1, outOffsets, ArrayStats.empty());
266+
return new VarBinArray(ctx.dtype(), n, outBytes, offsetArr, PType.I32, ArrayStats.empty());
267+
}
268+
269+
DType indicesDtype = new DType.Primitive(indicesPtype, false);
270+
Array indicesArray = decodeChild(ctx, 0, indicesDtype, numPatches);
271+
Array valuesArray = decodeChild(ctx, 1, ctx.dtype(), numPatches);
272+
273+
MemorySegment idxSeg = indicesArray.buffer(0);
274+
VarBinArray varBin = (VarBinArray) valuesArray;
275+
MemorySegment valBytes = varBin.buffer(0);
276+
MemorySegment valOffsets = varBin.child(0).buffer(0);
277+
PType valOffPtype = ((DType.Primitive) varBin.child(0).dtype()).ptype();
278+
279+
long totalBytes = 0;
280+
for (long i = 0; i < numPatches; i++) {
281+
totalBytes += readVarBinOffset(valOffsets, i + 1, valOffPtype)
282+
- readVarBinOffset(valOffsets, i, valOffPtype);
283+
}
284+
285+
MemorySegment outBytes = ctx.arena().allocate(Math.max(1, totalBytes));
286+
long patchCursor = 0;
287+
long bytePos = 0;
288+
for (long pos = 0; pos < n; pos++) {
289+
if (patchCursor < numPatches) {
290+
long patchPos = readUnsignedIdx(idxSeg, patchCursor, indicesPtype) - offset;
291+
if (patchPos == pos) {
292+
long strStart = readVarBinOffset(valOffsets, patchCursor, valOffPtype);
293+
long strEnd = readVarBinOffset(valOffsets, patchCursor + 1, valOffPtype);
294+
long strLen = strEnd - strStart;
295+
if (strLen > 0) {
296+
MemorySegment.copy(valBytes, strStart, outBytes, bytePos, strLen);
297+
bytePos += strLen;
298+
}
299+
patchCursor++;
300+
}
301+
}
302+
outOffsets.setAtIndex(PTypeIO.LE_INT, pos + 1, (int) bytePos);
303+
}
304+
305+
DType i32dtype = new DType.Primitive(PType.I32, false);
306+
Array offsetArr = new IntArray(i32dtype, n + 1, outOffsets, ArrayStats.empty());
307+
return new VarBinArray(ctx.dtype(), n, outBytes, offsetArr, PType.I32, ArrayStats.empty());
308+
}
309+
310+
private static long readVarBinOffset(MemorySegment seg, long i, PType ptype) {
311+
return switch (ptype) {
312+
case I32, U32 -> Integer.toUnsignedLong(seg.getAtIndex(PTypeIO.LE_INT, i));
313+
case I64, U64 -> seg.getAtIndex(PTypeIO.LE_LONG, i);
314+
default -> throw new VortexException(EncodingId.VORTEX_SPARSE, "unsupported offset ptype " + ptype);
315+
};
316+
}
317+
224318
private static Array decodeChild(DecodeContext parent, int childIdx, DType dtype, long rowCount) {
225319
ArrayNode childNode = parent.node().children()[childIdx];
226320
DecodeContext childCtx = new DecodeContext(
@@ -295,7 +389,7 @@ private static long scalarToLong(ScalarProtos.ScalarValue scalar, PType ptype) {
295389
case UINT64_VALUE -> scalar.getUint64Value();
296390
case F32_VALUE -> Float.floatToRawIntBits(scalar.getF32Value());
297391
case F64_VALUE -> Double.doubleToRawLongBits(scalar.getF64Value());
298-
case KIND_NOT_SET -> 0L;
392+
case NULL_VALUE, KIND_NOT_SET -> 0L;
299393
default -> throw new VortexException(EncodingId.VORTEX_SPARSE,
300394
"unexpected scalar kind " + scalar.getKindCase());
301395
};

‎core/src/test/java/io/github/dfa1/vortex/encoding/SparseEncodingTest.java‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
11
package io.github.dfa1.vortex.encoding;
22

33
import com.google.protobuf.InvalidProtocolBufferException;
4+
import com.google.protobuf.NullValue;
45
import io.github.dfa1.vortex.proto.DTypeProtos;
56
import io.github.dfa1.vortex.proto.EncodingProtos;
67
import io.github.dfa1.vortex.proto.ScalarProtos;
78
import io.github.dfa1.vortex.core.array.Array;
9+
import io.github.dfa1.vortex.core.array.BoolArray;
10+
import io.github.dfa1.vortex.core.array.VarBinArray;
811
import io.github.dfa1.vortex.core.ArrayStats;
912
import io.github.dfa1.vortex.core.DType;
1013
import io.github.dfa1.vortex.core.PType;
@@ -18,6 +21,7 @@
1821
import java.lang.foreign.ValueLayout;
1922
import java.nio.ByteBuffer;
2023
import java.nio.ByteOrder;
24+
import java.nio.charset.StandardCharsets;
2125
import java.util.List;
2226

2327
import static org.assertj.core.api.Assertions.assertThat;
@@ -212,6 +216,150 @@ void decode_offsetSubtracted() {
212216
assertThat(result.buffer(0).get(layout, 16L)).isEqualTo(777L);
213217
}
214218

219+
// regression: NULL_VALUE fill caused "unexpected scalar kind NULL_VALUE" on nullable cols
220+
@Test
221+
void decode_nullValueFill_treatedAsZero() {
222+
// Given — fill encoded as ScalarValue.NULL_VALUE (as Rust writes for nullable cols)
223+
byte[] nullFill = ScalarProtos.ScalarValue.newBuilder()
224+
.setNullValue(NullValue.NULL_VALUE).build().toByteArray();
225+
byte[] meta = buildSparseMetaBytes(0, 0L, PType.U32);
226+
DecodeContext ctx = buildCtx(I64_DTYPE, 4, nullFill, meta, new byte[0], new byte[0],
227+
new DType.Primitive(PType.U32, false));
228+
SparseEncoding sut = new SparseEncoding();
229+
230+
// When
231+
Array result = sut.decode(ctx);
232+
233+
// Then
234+
var layout = ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
235+
for (int i = 0; i < 4; i++) {
236+
assertThat(result.buffer(0).get(layout, (long) i * 8)).as("index %d", i).isZero();
237+
}
238+
}
239+
240+
// regression: Utf8 dtype caused "expected primitive dtype, got Utf8[nullable=true]"
241+
@Test
242+
void decode_utf8_noPatches_allEmpty() {
243+
// Given — Utf8 sparse, no patches → all positions empty (null fill)
244+
DType utf8 = new DType.Utf8(true);
245+
byte[] nullFill = ScalarProtos.ScalarValue.newBuilder()
246+
.setNullValue(NullValue.NULL_VALUE).build().toByteArray();
247+
byte[] meta = buildSparseMetaBytes(0, 0L, PType.U32);
248+
DecodeContext ctx = buildCtx(utf8, 3, nullFill, meta, new byte[0], new byte[0],
249+
new DType.Primitive(PType.U32, false));
250+
SparseEncoding sut = new SparseEncoding();
251+
252+
// When
253+
Array result = sut.decode(ctx);
254+
255+
// Then
256+
assertThat(result.length()).isEqualTo(3L);
257+
VarBinArray varBin = (VarBinArray) result;
258+
for (int i = 0; i < 3; i++) {
259+
assertThat(varBin.getByteLength(i)).as("index %d", i).isZero();
260+
}
261+
}
262+
263+
// regression: Utf8 dtype caused "expected primitive dtype, got Utf8[nullable=true]"
264+
@Test
265+
void decode_utf8_withPatches_writesStringsAtIndices() {
266+
// Given — 5 Utf8 elements, patches at [1]="hi" and [3]="bye"
267+
DType utf8 = new DType.Utf8(true);
268+
byte[] nullFill = ScalarProtos.ScalarValue.newBuilder()
269+
.setNullValue(NullValue.NULL_VALUE).build().toByteArray();
270+
byte[] meta = buildSparseMetaBytes(2, 0L, PType.U32);
271+
272+
byte[] idxBuf = toLEBytes(new long[]{1L, 3L}, PType.U32);
273+
byte[] strBytes = "hibye".getBytes(StandardCharsets.UTF_8);
274+
byte[] offsets = intLEBytes(new int[]{0, 2, 5});
275+
byte[] varBinMeta = EncodingProtos.VarBinMetadata.newBuilder()
276+
.setOffsetsPtype(DTypeProtos.PType.forNumber(PType.I32.ordinal()))
277+
.build().toByteArray();
278+
279+
ArrayNode offsetsNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null,
280+
new ArrayNode[0], new int[]{3}, ArrayStats.empty());
281+
ArrayNode valNode = new ArrayNode(EncodingId.VORTEX_VARBIN,
282+
ByteBuffer.wrap(varBinMeta),
283+
new ArrayNode[]{offsetsNode}, new int[]{2}, ArrayStats.empty());
284+
ArrayNode idxNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null,
285+
new ArrayNode[0], new int[]{1}, ArrayStats.empty());
286+
ArrayNode sparseNode = new ArrayNode(EncodingId.VORTEX_SPARSE,
287+
ByteBuffer.wrap(meta),
288+
new ArrayNode[]{idxNode, valNode}, new int[]{0}, ArrayStats.empty());
289+
290+
EncodingRegistry registry = EncodingRegistry.empty();
291+
registry.register(new SparseEncoding());
292+
registry.register(new PrimitiveEncoding());
293+
registry.register(new VarBinEncoding());
294+
295+
MemorySegment[] segments = {
296+
MemorySegment.ofArray(nullFill),
297+
MemorySegment.ofArray(idxBuf),
298+
MemorySegment.ofArray(strBytes),
299+
MemorySegment.ofArray(offsets),
300+
};
301+
DecodeContext ctx = new DecodeContext(sparseNode, utf8, 5, segments, registry, Arena.global());
302+
SparseEncoding sut = new SparseEncoding();
303+
304+
// When
305+
Array result = sut.decode(ctx);
306+
307+
// Then
308+
VarBinArray varBin = (VarBinArray) result;
309+
assertThat(varBin.length()).isEqualTo(5L);
310+
assertThat(varBin.getByteLength(0)).isZero();
311+
assertThat(new String(varBin.getBytes(1))).isEqualTo("hi");
312+
assertThat(varBin.getByteLength(2)).isZero();
313+
assertThat(new String(varBin.getBytes(3))).isEqualTo("bye");
314+
assertThat(varBin.getByteLength(4)).isZero();
315+
}
316+
317+
// regression: Bool dtype caused "expected primitive dtype, got Bool[nullable=true]"
318+
@Test
319+
void decode_bool_withPatches_setsBitsAtIndices() {
320+
// Given — 6 Bool elements, patches at [2]=true and [5]=true
321+
DType bool = new DType.Bool(true);
322+
byte[] nullFill = ScalarProtos.ScalarValue.newBuilder()
323+
.setNullValue(NullValue.NULL_VALUE).build().toByteArray();
324+
byte[] meta = buildSparseMetaBytes(2, 0L, PType.U32);
325+
byte[] idxBuf = toLEBytes(new long[]{2L, 5L}, PType.U32);
326+
byte[] boolBits = new byte[]{0b00000011};
327+
328+
ArrayNode valNode = new ArrayNode(EncodingId.VORTEX_BOOL, null,
329+
new ArrayNode[0], new int[]{2}, ArrayStats.empty());
330+
ArrayNode idxNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null,
331+
new ArrayNode[0], new int[]{1}, ArrayStats.empty());
332+
ArrayNode sparseNode = new ArrayNode(EncodingId.VORTEX_SPARSE,
333+
ByteBuffer.wrap(meta),
334+
new ArrayNode[]{idxNode, valNode}, new int[]{0}, ArrayStats.empty());
335+
336+
EncodingRegistry registry = EncodingRegistry.empty();
337+
registry.register(new SparseEncoding());
338+
registry.register(new PrimitiveEncoding());
339+
registry.register(new BoolEncoding());
340+
341+
MemorySegment[] segments = {
342+
MemorySegment.ofArray(nullFill),
343+
MemorySegment.ofArray(idxBuf),
344+
MemorySegment.ofArray(boolBits),
345+
};
346+
DecodeContext ctx = new DecodeContext(sparseNode, bool, 6, segments, registry, Arena.global());
347+
SparseEncoding sut = new SparseEncoding();
348+
349+
// When
350+
Array result = sut.decode(ctx);
351+
352+
// Then
353+
BoolArray boolArr = (BoolArray) result;
354+
assertThat(boolArr.length()).isEqualTo(6L);
355+
assertThat(boolArr.getBoolean(0)).isFalse();
356+
assertThat(boolArr.getBoolean(1)).isFalse();
357+
assertThat(boolArr.getBoolean(2)).isTrue();
358+
assertThat(boolArr.getBoolean(3)).isFalse();
359+
assertThat(boolArr.getBoolean(4)).isFalse();
360+
assertThat(boolArr.getBoolean(5)).isTrue();
361+
}
362+
215363
private static DecodeContext buildSparseCtx(
216364
DType dtype, long rowCount, long fillLong, PType idxPtype,
217365
long[] patchIndices, long[] patchValues
@@ -315,5 +463,14 @@ private static byte[] f64LEBytes(double[] values) {
315463
}
316464
return buf;
317465
}
466+
467+
private static byte[] intLEBytes(int[] values) {
468+
byte[] buf = new byte[values.length * 4];
469+
ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN);
470+
for (int v : values) {
471+
bb.putInt(v);
472+
}
473+
return buf;
474+
}
318475
}
319476
}

0 commit comments

Comments
 (0)