Skip to content

Commit 05cedb7

Browse files
dfa1claude
dfa1
and
claude
committed
revert(reader): restore ArrayNode Known/Unknown split
Reverts 3b25a97. Also restores KnownArrayNode/UnknownArrayNode, whose deletions were swept into the unrelated dict-lane perf commit 12e1350 instead of the collapse commit, and drops the collapse entry from the unreleased changelog section. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent 8447c64 commit 05cedb7

9 files changed

Lines changed: 114 additions & 52 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1717
- `Compute.filteredSum` over a dictionary-encoded filter column is ~20× faster (best runs ~30×): the predicate is resolved against the dictionary's value pool once and the raw `u8` codes are scanned directly from their backing segments, instead of decoding every row through the per-element accessor — a fused `SUM(measure) WHERE category = …` over 100M rows drops from ~760 ms to ~38 ms. ([85e251cc](https://github.com/dfa1/vortex-java/commit/85e251cc))
1818
- `Compute.filteredAggregate` takes the same dictionary code-scan lane (`COUNT(*)` included) — ~22× faster on the same workload (~980 ms → ~46 ms over 100M rows), which the Calcite `WHERE`-filtered aggregate push-down inherits on its boundary chunks. ([145791c7](https://github.com/dfa1/vortex-java/commit/145791c7), [6e6d7dd0](https://github.com/dfa1/vortex-java/commit/6e6d7dd0))
1919
- A multi-column `AND` filter no longer forfeits the dictionary lane: the dict-encoded leaf drives the code scan and the remaining predicates are evaluated only on its matches — `SUM(…) WHERE category = 7 AND price > 500` over 100M rows drops from ~2.3 s to ~200 ms (~11×). ([12e13501](https://github.com/dfa1/vortex-java/commit/12e13501))
20-
- `reader.decode.ArrayNode` is now a single record carrying the raw encoding-id string from the wire; the `KnownArrayNode` / `UnknownArrayNode` split is gone. Custom `EncodingDecoder` implementations keep working unchanged — decode dispatch is keyed by the id string, and the `allowUnknown` passthrough behaves as before. ([3b25a97d](https://github.com/dfa1/vortex-java/commit/3b25a97d))
2120

2221
## [0.11.0] — 2026-06-28
2322

‎reader/src/main/java/io/github/dfa1/vortex/reader/FlatSegmentDecoder.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,12 @@
66
import io.github.dfa1.vortex.core.model.DType;
77
import io.github.dfa1.vortex.core.io.IoBounds;
88
import io.github.dfa1.vortex.reader.array.Array;
9+
import io.github.dfa1.vortex.core.model.EncodingId;
910
import io.github.dfa1.vortex.core.fbs.FbsBuffer;
1011
import io.github.dfa1.vortex.reader.decode.ArrayNode;
1112
import io.github.dfa1.vortex.reader.decode.DecodeContext;
13+
import io.github.dfa1.vortex.reader.decode.KnownArrayNode;
14+
import io.github.dfa1.vortex.reader.decode.UnknownArrayNode;
1215

1316
import java.lang.foreign.MemorySegment;
1417
import java.lang.foreign.SegmentAllocator;
@@ -96,6 +99,8 @@ private static ArrayNode convertArrayNode(
9699
}
97100

98101
MemorySegment meta = fbs.metadataAsSegment();
99-
return new ArrayNode(rawEncodingId, meta, children, bufferIndices);
102+
return EncodingId.parse(rawEncodingId)
103+
.<ArrayNode>map(known -> new KnownArrayNode(known, meta, children, bufferIndices))
104+
.orElseGet(() -> new UnknownArrayNode(rawEncodingId, meta, children, bufferIndices));
100105
}
101106
}

‎reader/src/main/java/io/github/dfa1/vortex/reader/ReadRegistry.java‎

Lines changed: 32 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,12 @@
77
import io.github.dfa1.vortex.reader.decode.ArrayNode;
88
import io.github.dfa1.vortex.reader.decode.DecodeContext;
99
import io.github.dfa1.vortex.reader.decode.EncodingDecoder;
10+
import io.github.dfa1.vortex.reader.decode.KnownArrayNode;
11+
import io.github.dfa1.vortex.reader.decode.UnknownArrayNode;
1012

1113
import java.lang.foreign.MemorySegment;
1214
import java.util.Collections;
15+
import java.util.Comparator;
1316
import java.util.Map;
1417
import java.util.ServiceLoader;
1518
import java.util.TreeMap;
@@ -20,15 +23,15 @@
2023
/// via the [#loadAll()] and [#empty()] convenience factories.
2124
public final class ReadRegistry {
2225

23-
private final Map<String, EncodingDecoder> decoders;
26+
private final Map<EncodingId, EncodingDecoder> decoders;
2427
private final boolean allowUnknown;
2528

2629
private ReadRegistry(Map<EncodingId, EncodingDecoder> decoders, boolean allowUnknown) {
27-
// Keyed by the id's wire string — the same form ArrayNode carries — so decode dispatch is
28-
// one map hit with no enum resolution. TreeMap keeps the stable name order mirroring
29-
// WriteRegistry.
30-
var sorted = new TreeMap<String, EncodingDecoder>();
31-
decoders.forEach((id, decoder) -> sorted.put(id.id(), decoder));
30+
// Order by encoding name, mirroring WriteRegistry (Enum.compareTo is final, so a Comparator
31+
// is required to sort by id rather than ordinal). Decode dispatch is keyed, so order is not
32+
// load-bearing here, but a stable order keeps the two registries consistent.
33+
var sorted = new TreeMap<EncodingId, EncodingDecoder>(Comparator.comparing(EncodingId::id));
34+
sorted.putAll(decoders);
3235
this.decoders = Collections.unmodifiableMap(sorted);
3336
this.allowUnknown = allowUnknown;
3437
}
@@ -67,7 +70,7 @@ public boolean isAllowUnknown() {
6770
/// @param encodingId the encoding id to query
6871
/// @return `true` if a decoder is registered
6972
public boolean hasDecoder(EncodingId encodingId) {
70-
return decoders.containsKey(encodingId.id());
73+
return decoders.containsKey(encodingId);
7174
}
7275

7376
/// Decodes the array described by `ctx`.
@@ -76,14 +79,21 @@ public boolean hasDecoder(EncodingId encodingId) {
7679
/// @return the decoded [Array]
7780
public Array decode(DecodeContext ctx) {
7881
ArrayNode node = ctx.node();
79-
EncodingDecoder decoder = decoders.get(node.encodingId());
82+
EncodingDecoder decoder = switch (node) {
83+
case KnownArrayNode k -> decoders.get(k.encodingId());
84+
case UnknownArrayNode _ -> null;
85+
};
8086
if (decoder != null) {
8187
return decoder.decode(ctx);
8288
}
8389
if (allowUnknown) {
8490
return decodeUnknown(ctx, node);
8591
}
86-
throw new VortexException("no decoder registered for " + node.encodingId());
92+
String id = switch (node) {
93+
case KnownArrayNode k -> k.encodingId().id();
94+
case UnknownArrayNode u -> u.rawEncodingId();
95+
};
96+
throw new VortexException("no decoder registered for " + id);
8797
}
8898

8999
/// Decodes the array described by `ctx` and returns its primary backing segment.
@@ -92,16 +102,25 @@ public Array decode(DecodeContext ctx) {
92102
/// @return the primary [MemorySegment] of the decoded array
93103
public MemorySegment decodeAsSegment(DecodeContext ctx) {
94104
ArrayNode node = ctx.node();
95-
EncodingDecoder decoder = decoders.get(node.encodingId());
105+
EncodingDecoder decoder = switch (node) {
106+
case KnownArrayNode k -> decoders.get(k.encodingId());
107+
case UnknownArrayNode _ -> null;
108+
};
96109
if (decoder != null) {
97110
return decoder.decode(ctx).materialize(ctx.arena());
98111
}
99-
throw new VortexException("no decoder registered for " + node.encodingId()
100-
+ " (or encoding has no primary segment)");
112+
String id = switch (node) {
113+
case KnownArrayNode k -> k.encodingId().id();
114+
case UnknownArrayNode u -> u.rawEncodingId();
115+
};
116+
throw new VortexException("no decoder registered for " + id + " (or encoding has no primary segment)");
101117
}
102118

103119
private static UnknownArray decodeUnknown(DecodeContext ctx, ArrayNode node) {
104-
String rawId = node.encodingId();
120+
String rawId = switch (node) {
121+
case KnownArrayNode k -> k.encodingId().id();
122+
case UnknownArrayNode u -> u.rawEncodingId();
123+
};
105124
MemorySegment[] bufs = new MemorySegment[node.bufferIndices().length];
106125
for (int i = 0; i < bufs.length; i++) {
107126
bufs[i] = ctx.buffer(i);

‎reader/src/main/java/io/github/dfa1/vortex/reader/decode/ArrayNode.java‎

Lines changed: 21 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -4,38 +4,34 @@
44

55
import java.lang.foreign.MemorySegment;
66

7-
/// Encoded array node as stored in a Flat layout segment — the in-file representation before
8-
/// decoding.
7+
/// Encoded array node as stored in a Flat layout segment.
8+
/// In-file representation before decoding; mirrors the Go ArrayNode struct.
99
///
10-
/// The encoding id is carried as the raw string from the wire, because encoding ids ARE strings in
11-
/// the format (`"vortex.flat"`, `"fastlanes.bitpacked"`, …). Whether an id is decodable is the
12-
/// [io.github.dfa1.vortex.reader.ReadRegistry]'s question at decode time, not a property of the
13-
/// node: an id no registered decoder claims either decodes as a passthrough
14-
/// [io.github.dfa1.vortex.reader.array.UnknownArray] (when
15-
/// [io.github.dfa1.vortex.reader.ReadRegistry#isAllowUnknown()] is set) or fails the decode.
16-
///
17-
/// @param encodingId the raw encoding id string from the file
18-
/// @param metadata encoding-specific metadata bytes, or `null`
19-
/// @param children child nodes
20-
/// @param bufferIndices segment buffer indices for this node
21-
@SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared.
22-
public record ArrayNode(
23-
String encodingId,
24-
MemorySegment metadata,
25-
ArrayNode[] children,
26-
int[] bufferIndices
27-
) {
10+
/// Sealed: a node is either [KnownArrayNode] (id resolves to an [EncodingId]) or
11+
/// [UnknownArrayNode] (id is an arbitrary string only meaningful for
12+
/// [io.github.dfa1.vortex.reader.ReadRegistry#isAllowUnknown()] passthrough decode).
13+
public sealed interface ArrayNode permits KnownArrayNode, UnknownArrayNode {
2814

29-
/// Short factory for the common case: a node whose encoding id is well-known. Mostly used by
30-
/// tests and helper code that converts an `EncodeNode` tree back into an `ArrayNode` tree.
15+
/// Short factory for the common case: a node whose encoding id is well-known.
16+
/// Mostly used by tests and helper code that converts an `EncodeNode` tree back into
17+
/// an `ArrayNode` tree.
3118
///
3219
/// @param encodingId the well-known encoding identifier
3320
/// @param metadata encoding-specific metadata bytes, or `null`
3421
/// @param children child nodes
3522
/// @param bufferIndices segment buffer indices for this node
36-
/// @return an [ArrayNode] carrying the id's wire string
37-
public static ArrayNode of(EncodingId encodingId, MemorySegment metadata, ArrayNode[] children,
23+
/// @return a [KnownArrayNode] with the given fields
24+
static ArrayNode of(EncodingId encodingId, MemorySegment metadata, ArrayNode[] children,
3825
int[] bufferIndices) {
39-
return new ArrayNode(encodingId.id(), metadata, children, bufferIndices);
26+
return new KnownArrayNode(encodingId, metadata, children, bufferIndices);
4027
}
28+
29+
/// @return encoding-specific metadata bytes, or `null`
30+
MemorySegment metadata();
31+
32+
/// @return child nodes
33+
ArrayNode[] children();
34+
35+
/// @return segment buffer indices for this node
36+
int[] bufferIndices();
4137
}

‎reader/src/main/java/io/github/dfa1/vortex/reader/decode/KnownArrayNode.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package io.github.dfa1.vortex.reader.decode;
2+
3+
import io.github.dfa1.vortex.core.model.EncodingId;
4+
5+
import java.lang.foreign.MemorySegment;
6+
7+
/// Array node whose encoding id is well-known to this build (an [EncodingId] enum constant).
8+
///
9+
/// @param encodingId well-known encoding id
10+
/// @param metadata encoding-specific metadata bytes, or `null`
11+
/// @param children child nodes
12+
/// @param bufferIndices segment buffer indices
13+
@SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared.
14+
public record KnownArrayNode(
15+
EncodingId encodingId,
16+
MemorySegment metadata,
17+
ArrayNode[] children,
18+
int[] bufferIndices
19+
) implements ArrayNode {
20+
}

‎reader/src/main/java/io/github/dfa1/vortex/reader/decode/UnknownArrayNode.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package io.github.dfa1.vortex.reader.decode;
2+
3+
import java.lang.foreign.MemorySegment;
4+
5+
/// Array node whose encoding id is not a recognized [io.github.dfa1.vortex.core.model.EncodingId].
6+
/// Produced when a file uses an encoding this build does not know about. Decoded as
7+
/// [io.github.dfa1.vortex.reader.array.UnknownArray] when
8+
/// [io.github.dfa1.vortex.reader.ReadRegistry#isAllowUnknown()] is set; otherwise the decode call throws.
9+
///
10+
/// @param rawEncodingId the raw encoding id string from the file
11+
/// @param metadata encoding-specific metadata bytes, or `null`
12+
/// @param children child nodes
13+
/// @param bufferIndices segment buffer indices
14+
@SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared.
15+
public record UnknownArrayNode(
16+
String rawEncodingId,
17+
MemorySegment metadata,
18+
ArrayNode[] children,
19+
int[] bufferIndices
20+
) implements ArrayNode {
21+
}

‎reader/src/test/java/io/github/dfa1/vortex/reader/FlatSegmentDecoderDecodeTest.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,11 @@
1717

1818
/// Successful flat-segment decode path — complements [FlatSegmentBoundsSecurityTest] (which only
1919
/// drives the rejection paths). A buffer descriptor with non-zero padding exercises the offset
20-
/// walk, and an unknown encoding id exercises the unknown-id passthrough through an
20+
/// walk, and an unknown encoding id exercises the `UnknownArrayNode` fallback through an
2121
/// allow-unknown registry. Together these pin two otherwise-untested spots:
2222
/// - the `dataOffset += padding` accumulation: with padding > 0, flipping `+=` to `-=` slices at a
2323
/// negative offset and fails, so a clean decode proves the addition.
24-
/// - the unknown-id node construction: mishandling an unresolvable id there yields a
24+
/// - the `orElseGet(() -> new UnknownArrayNode(...))` fallback: returning `null` there yields a
2525
/// null node and the decode would not produce an `UnknownArray`.
2626
class FlatSegmentDecoderDecodeTest {
2727

@@ -46,7 +46,7 @@ void decode_unknownEncodingWithBufferPadding_returnsUnknownArray() {
4646
DType.I32, 0, arena);
4747

4848
// Then — the allow-unknown path produced an UnknownArray (proves both the +padding
49-
// walk and the unknown-id passthrough ran)
49+
// walk and the UnknownArrayNode fallback ran)
5050
assertThat(result).isInstanceOf(UnknownArray.class);
5151
}
5252
}

‎reader/src/test/java/io/github/dfa1/vortex/reader/ReadRegistryTest.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import io.github.dfa1.vortex.reader.decode.ArrayNode;
99
import io.github.dfa1.vortex.reader.decode.DecodeContext;
1010
import io.github.dfa1.vortex.reader.decode.EncodingDecoder;
11+
import io.github.dfa1.vortex.reader.decode.UnknownArrayNode;
1112
import org.junit.jupiter.api.Test;
1213

1314
import java.lang.foreign.Arena;
@@ -24,7 +25,7 @@ class ReadRegistryTest {
2425
void decodeUnknownEncodingThrowsByDefault() {
2526
// Given
2627
ReadRegistry sut = ReadRegistry.empty();
27-
ArrayNode node = new ArrayNode("some.unknown",
28+
ArrayNode node = new UnknownArrayNode("some.unknown",
2829
MemorySegment.ofArray(new byte[0]), new ArrayNode[0], new int[0]);
2930
DecodeContext ctx = new DecodeContext(node, DTypes.I32, 0L,
3031
new MemorySegment[0], sut, Arena.ofAuto());
@@ -74,7 +75,7 @@ void decodeUnknownEncodingReturnsUnknownArrayWhenAllowed() {
7475
MemorySegment metadata = MemorySegment.ofArray(new byte[]{1, 2, 3});
7576
MemorySegment buf = Arena.ofAuto().allocate(4);
7677
buf.set(java.lang.foreign.ValueLayout.JAVA_INT, 0, 42);
77-
ArrayNode node = new ArrayNode("some.unknown",
78+
ArrayNode node = new UnknownArrayNode("some.unknown",
7879
metadata, new ArrayNode[0], new int[]{0});
7980
DecodeContext ctx = new DecodeContext(node, DTypes.I32, 5L,
8081
new MemorySegment[]{buf}, sut, Arena.ofAuto());
@@ -102,7 +103,7 @@ void decodeUnknownEncodingWrapsChildrenAsUnknown() {
102103
// its parent is unknown — mirrors Rust decode_foreign in vortex-array/src/serde.rs:380.
103104
ArrayNode child = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE,
104105
MemorySegment.ofArray(new byte[0]), new ArrayNode[0], new int[0]);
105-
ArrayNode parent = new ArrayNode("some.unknown",
106+
ArrayNode parent = new UnknownArrayNode("some.unknown",
106107
MemorySegment.ofArray(new byte[0]), new ArrayNode[]{child}, new int[0]);
107108
DecodeContext ctx = new DecodeContext(parent, DTypes.I32, 0L,
108109
new MemorySegment[0], sut, Arena.ofAuto());

‎writer/src/test/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoderTest.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import io.github.dfa1.vortex.reader.decode.DecodeContext;
1010

1111
import io.github.dfa1.vortex.core.model.EncodingId;
12+
import io.github.dfa1.vortex.reader.decode.KnownArrayNode;
1213
import io.github.dfa1.vortex.reader.ReadRegistry;
1314
import io.github.dfa1.vortex.reader.decode.TestRegistry;
1415
import io.github.dfa1.vortex.core.proto.ProtoRLEMetadata;
@@ -33,12 +34,12 @@ class RleEncodingEncoderTest {
3334
private static final RleEncodingDecoder DECODER = new RleEncodingDecoder();
3435
private static final ReadRegistry REGISTRY = TestRegistry.ofDecoders(DECODER, new PrimitiveEncodingDecoder());
3536

36-
private static ArrayNode toArrayNode(EncodeNode enc) {
37+
private static KnownArrayNode toArrayNode(EncodeNode enc) {
3738
ArrayNode[] children = new ArrayNode[enc.children().length];
3839
for (int i = 0; i < children.length; i++) {
3940
children[i] = toArrayNode(enc.children()[i]);
4041
}
41-
return ArrayNode.of(enc.encodingId(), enc.metadata(), children, enc.bufferIndices());
42+
return new KnownArrayNode(enc.encodingId(), enc.metadata(), children, enc.bufferIndices());
4243
}
4344

4445
@Nested
@@ -267,14 +268,14 @@ void decode_nullableIndices_returnsMaskedArrayWithCorrectValidity() {
267268
originalBufs.add(validityBuf);
268269
MemorySegment[] segments = originalBufs.toArray(new MemorySegment[0]);
269270

270-
ArrayNode origRoot = toArrayNode(encoded.rootNode());
271-
ArrayNode origIndices = origRoot.children()[1];
271+
KnownArrayNode origRoot = toArrayNode(encoded.rootNode());
272+
KnownArrayNode origIndices = (KnownArrayNode) origRoot.children()[1];
272273
ArrayNode validityNode = ArrayNode.of(
273274
EncodingId.VORTEX_BOOL, null, new ArrayNode[0], new int[]{3});
274-
ArrayNode nullableIndices = new ArrayNode(
275+
ArrayNode nullableIndices = ArrayNode.of(
275276
origIndices.encodingId(), origIndices.metadata(),
276277
new ArrayNode[]{validityNode}, origIndices.bufferIndices());
277-
ArrayNode root = new ArrayNode(
278+
ArrayNode root = ArrayNode.of(
278279
origRoot.encodingId(), origRoot.metadata(),
279280
new ArrayNode[]{origRoot.children()[0], nullableIndices, origRoot.children()[2]},
280281
origRoot.bufferIndices());

0 commit comments

Comments
 (0)