From 8447c64dfd250cbac25b78bcaa7ee0641419b9e1 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 4 Jul 2026 08:56:28 +0200 Subject: [PATCH 1/7] refactor(reader): ExtensionDecoder to extension pkg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the ExtensionDecoder interface from the reader root into reader.extension next to its four implementations, mirroring EncodingDecoder in reader.decode. Also sync stale docs: ReadRegistry no longer manages extension decoders (no register(ExtensionDecoder), no ServiceLoader discovery) — the spec set is closed and dispatched in Chunk.as(). Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 12 ++++++------ docs/reference.md | 10 +++++----- .../reader/extension/DateExtensionDecoder.java | 1 - .../reader/{ => extension}/ExtensionDecoder.java | 2 +- .../reader/extension/TimeExtensionDecoder.java | 1 - .../reader/extension/TimestampExtensionDecoder.java | 1 - .../reader/extension/UuidExtensionDecoder.java | 1 - 7 files changed, 12 insertions(+), 16 deletions(-) rename reader/src/main/java/io/github/dfa1/vortex/reader/{ => extension}/ExtensionDecoder.java (93%) diff --git a/CLAUDE.md b/CLAUDE.md index 21ed1d34..d30b9246 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,12 +28,12 @@ core — everything lives under `io.github.dfa1.vortex.core.*`: core.error VortexException core.compute FastLanes, PrimitiveArrays core.fbs / core.proto — generated wire codecs + their runtimes -reader — VortexReader, VortexHttpReader, VortexHandle, ReadRegistry, ExtensionDecoder, - Chunk, ArrayStats, ScanOptions, RowFilter; file internals (Footer, Layout, Trailer, +reader — VortexReader, VortexHttpReader, VortexHandle, ReadRegistry, Chunk, ArrayStats, + ScanOptions, RowFilter; file internals (Footer, Layout, Trailer, PostscriptParser, …) reader.array — Array + all subtypes (decode outputs) reader.decode — EncodingDecoder, DecodeContext, ArrayNode + *EncodingDecoder impls - reader.extension — Date/Time/Timestamp/Uuid ExtensionDecoder + reader.extension — ExtensionDecoder + Date/Time/Timestamp/Uuid impls writer — VortexWriter, WriteRegistry, WriteOptions, ExtensionEncoder writer.encode — EncodingEncoder, EncodeContext, NullableData + *EncodingEncoder impls, extension encoders @@ -135,9 +135,9 @@ Add `EncodingId` enum constant `VORTEX_FOO("vortex.foo")`, then per side: ### Adding an extension type Add `ExtensionId` constant, then per side: -- **Decode:** `FooExtensionDecoder implements ExtensionDecoder` in `reader.extension`; register via - `ReadRegistry.builder().register(new FooExtensionDecoder())` — **no service file** - (`registerServiceLoaded()` does not discover `ExtensionDecoder`). +- **Decode:** singleton `FooExtensionDecoder implements ExtensionDecoder` in `reader.extension` + + a `case VORTEX_FOO` in `Chunk.as()` — not registry-managed, **no service file** + (`registerServiceLoaded()` only discovers `EncodingDecoder`). - **Encode:** `FooExtensionEncoder implements ExtensionEncoder` in `writer` + FQN in `writer/.../META-INF/services/io.github.dfa1.vortex.writer.ExtensionEncoder` diff --git a/docs/reference.md b/docs/reference.md index 7d1d81c5..ede62dbf 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -203,14 +203,14 @@ Immutable after construction. Build via `ReadRegistry.builder()` or the static c | Method | Notes | |------------------------------|------------------------------------------------------------------------------------------| | `register(EncodingDecoder)` | Add a custom encoding decoder; throws if already registered | -| `register(ExtensionDecoder)` | Add a custom extension decoder; throws if already registered | -| `registerServiceLoaded()` | Add every `EncodingDecoder` and `ExtensionDecoder` discovered via `ServiceLoader` | +| `registerServiceLoaded()` | Add every `EncodingDecoder` discovered via `ServiceLoader` | | `allowUnknown()` | Switch to passthrough mode — unknown nodes (and their children) decode as `UnknownArray` | | `build()` | Produce the immutable `ReadRegistry` | -Register custom decoders via `ServiceLoader` by adding the fully qualified class name to -`META-INF/services/io.github.dfa1.vortex.reader.decode.EncodingDecoder` or -`META-INF/services/io.github.dfa1.vortex.reader.ExtensionDecoder`. +Register custom encoding decoders via `ServiceLoader` by adding the fully qualified class name to +`META-INF/services/io.github.dfa1.vortex.reader.decode.EncodingDecoder`. Extension decoders +(`io.github.dfa1.vortex.reader.extension.ExtensionDecoder`) are not registry-managed: the built-in +implementations are singletons invoked directly by their `ExtensionId`. --- diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoder.java index eb4c40cc..653334b8 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/DateExtensionDecoder.java @@ -8,7 +8,6 @@ import io.github.dfa1.vortex.reader.array.MaskedArray; import io.github.dfa1.vortex.core.model.ExtensionId; -import io.github.dfa1.vortex.reader.ExtensionDecoder; import java.time.LocalDate; import java.util.ArrayList; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ExtensionDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/ExtensionDecoder.java similarity index 93% rename from reader/src/main/java/io/github/dfa1/vortex/reader/ExtensionDecoder.java rename to reader/src/main/java/io/github/dfa1/vortex/reader/extension/ExtensionDecoder.java index 3f4a94ce..ed4d4214 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ExtensionDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/ExtensionDecoder.java @@ -1,4 +1,4 @@ -package io.github.dfa1.vortex.reader; +package io.github.dfa1.vortex.reader.extension; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.ExtensionId; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/TimeExtensionDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/TimeExtensionDecoder.java index 6857a4c1..9befb5f7 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/TimeExtensionDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/TimeExtensionDecoder.java @@ -9,7 +9,6 @@ import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.TimeDtype; -import io.github.dfa1.vortex.reader.ExtensionDecoder; import java.time.LocalTime; import java.util.ArrayList; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/TimestampExtensionDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/TimestampExtensionDecoder.java index fe5a0be6..879976cd 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/TimestampExtensionDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/TimestampExtensionDecoder.java @@ -9,7 +9,6 @@ import io.github.dfa1.vortex.core.model.ExtensionId; import io.github.dfa1.vortex.core.model.TimestampDtype; -import io.github.dfa1.vortex.reader.ExtensionDecoder; import java.time.Instant; import java.time.ZoneId; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/UuidExtensionDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/UuidExtensionDecoder.java index f82b7bd4..433223a1 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/extension/UuidExtensionDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/extension/UuidExtensionDecoder.java @@ -9,7 +9,6 @@ import io.github.dfa1.vortex.reader.array.MaskedArray; import io.github.dfa1.vortex.core.model.ExtensionId; -import io.github.dfa1.vortex.reader.ExtensionDecoder; import java.util.ArrayList; import java.util.List; From 05cedb7ed370e48cf2aec2d51d60f433ab92e40b Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 4 Jul 2026 09:05:30 +0200 Subject: [PATCH 2/7] revert(reader): restore ArrayNode Known/Unknown split Reverts 3b25a97d. Also restores KnownArrayNode/UnknownArrayNode, whose deletions were swept into the unrelated dict-lane perf commit 12e13501 instead of the collapse commit, and drops the collapse entry from the unreleased changelog section. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 1 - .../vortex/reader/FlatSegmentDecoder.java | 7 ++- .../dfa1/vortex/reader/ReadRegistry.java | 45 ++++++++++++------ .../dfa1/vortex/reader/decode/ArrayNode.java | 46 +++++++++---------- .../vortex/reader/decode/KnownArrayNode.java | 20 ++++++++ .../reader/decode/UnknownArrayNode.java | 21 +++++++++ .../reader/FlatSegmentDecoderDecodeTest.java | 6 +-- .../dfa1/vortex/reader/ReadRegistryTest.java | 7 +-- .../writer/encode/RleEncodingEncoderTest.java | 13 +++--- 9 files changed, 114 insertions(+), 52 deletions(-) create mode 100644 reader/src/main/java/io/github/dfa1/vortex/reader/decode/KnownArrayNode.java create mode 100644 reader/src/main/java/io/github/dfa1/vortex/reader/decode/UnknownArrayNode.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ec90135..5718ba5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `Compute.filteredSum` over a dictionary-encoded filter column is ~20× faster (best runs ~30×): the predicate is resolved against the dictionary's value pool once and the raw `u8` codes are scanned directly from their backing segments, instead of decoding every row through the per-element accessor — a fused `SUM(measure) WHERE category = …` over 100M rows drops from ~760 ms to ~38 ms. ([85e251cc](https://github.com/dfa1/vortex-java/commit/85e251cc)) - `Compute.filteredAggregate` takes the same dictionary code-scan lane (`COUNT(*)` included) — ~22× faster on the same workload (~980 ms → ~46 ms over 100M rows), which the Calcite `WHERE`-filtered aggregate push-down inherits on its boundary chunks. ([145791c7](https://github.com/dfa1/vortex-java/commit/145791c7), [6e6d7dd0](https://github.com/dfa1/vortex-java/commit/6e6d7dd0)) - A multi-column `AND` filter no longer forfeits the dictionary lane: the dict-encoded leaf drives the code scan and the remaining predicates are evaluated only on its matches — `SUM(…) WHERE category = 7 AND price > 500` over 100M rows drops from ~2.3 s to ~200 ms (~11×). ([12e13501](https://github.com/dfa1/vortex-java/commit/12e13501)) -- `reader.decode.ArrayNode` is now a single record carrying the raw encoding-id string from the wire; the `KnownArrayNode` / `UnknownArrayNode` split is gone. Custom `EncodingDecoder` implementations keep working unchanged — decode dispatch is keyed by the id string, and the `allowUnknown` passthrough behaves as before. ([3b25a97d](https://github.com/dfa1/vortex-java/commit/3b25a97d)) ## [0.11.0] — 2026-06-28 diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/FlatSegmentDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/FlatSegmentDecoder.java index fab481f8..6fe9d8cd 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/FlatSegmentDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/FlatSegmentDecoder.java @@ -6,9 +6,12 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.io.IoBounds; import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.fbs.FbsBuffer; import io.github.dfa1.vortex.reader.decode.ArrayNode; import io.github.dfa1.vortex.reader.decode.DecodeContext; +import io.github.dfa1.vortex.reader.decode.KnownArrayNode; +import io.github.dfa1.vortex.reader.decode.UnknownArrayNode; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -96,6 +99,8 @@ private static ArrayNode convertArrayNode( } MemorySegment meta = fbs.metadataAsSegment(); - return new ArrayNode(rawEncodingId, meta, children, bufferIndices); + return EncodingId.parse(rawEncodingId) + .map(known -> new KnownArrayNode(known, meta, children, bufferIndices)) + .orElseGet(() -> new UnknownArrayNode(rawEncodingId, meta, children, bufferIndices)); } } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ReadRegistry.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ReadRegistry.java index 144035eb..9faa572d 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ReadRegistry.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ReadRegistry.java @@ -7,9 +7,12 @@ import io.github.dfa1.vortex.reader.decode.ArrayNode; import io.github.dfa1.vortex.reader.decode.DecodeContext; import io.github.dfa1.vortex.reader.decode.EncodingDecoder; +import io.github.dfa1.vortex.reader.decode.KnownArrayNode; +import io.github.dfa1.vortex.reader.decode.UnknownArrayNode; import java.lang.foreign.MemorySegment; import java.util.Collections; +import java.util.Comparator; import java.util.Map; import java.util.ServiceLoader; import java.util.TreeMap; @@ -20,15 +23,15 @@ /// via the [#loadAll()] and [#empty()] convenience factories. public final class ReadRegistry { - private final Map decoders; + private final Map decoders; private final boolean allowUnknown; private ReadRegistry(Map decoders, boolean allowUnknown) { - // Keyed by the id's wire string — the same form ArrayNode carries — so decode dispatch is - // one map hit with no enum resolution. TreeMap keeps the stable name order mirroring - // WriteRegistry. - var sorted = new TreeMap(); - decoders.forEach((id, decoder) -> sorted.put(id.id(), decoder)); + // Order by encoding name, mirroring WriteRegistry (Enum.compareTo is final, so a Comparator + // is required to sort by id rather than ordinal). Decode dispatch is keyed, so order is not + // load-bearing here, but a stable order keeps the two registries consistent. + var sorted = new TreeMap(Comparator.comparing(EncodingId::id)); + sorted.putAll(decoders); this.decoders = Collections.unmodifiableMap(sorted); this.allowUnknown = allowUnknown; } @@ -67,7 +70,7 @@ public boolean isAllowUnknown() { /// @param encodingId the encoding id to query /// @return `true` if a decoder is registered public boolean hasDecoder(EncodingId encodingId) { - return decoders.containsKey(encodingId.id()); + return decoders.containsKey(encodingId); } /// Decodes the array described by `ctx`. @@ -76,14 +79,21 @@ public boolean hasDecoder(EncodingId encodingId) { /// @return the decoded [Array] public Array decode(DecodeContext ctx) { ArrayNode node = ctx.node(); - EncodingDecoder decoder = decoders.get(node.encodingId()); + EncodingDecoder decoder = switch (node) { + case KnownArrayNode k -> decoders.get(k.encodingId()); + case UnknownArrayNode _ -> null; + }; if (decoder != null) { return decoder.decode(ctx); } if (allowUnknown) { return decodeUnknown(ctx, node); } - throw new VortexException("no decoder registered for " + node.encodingId()); + String id = switch (node) { + case KnownArrayNode k -> k.encodingId().id(); + case UnknownArrayNode u -> u.rawEncodingId(); + }; + throw new VortexException("no decoder registered for " + id); } /// Decodes the array described by `ctx` and returns its primary backing segment. @@ -92,16 +102,25 @@ public Array decode(DecodeContext ctx) { /// @return the primary [MemorySegment] of the decoded array public MemorySegment decodeAsSegment(DecodeContext ctx) { ArrayNode node = ctx.node(); - EncodingDecoder decoder = decoders.get(node.encodingId()); + EncodingDecoder decoder = switch (node) { + case KnownArrayNode k -> decoders.get(k.encodingId()); + case UnknownArrayNode _ -> null; + }; if (decoder != null) { return decoder.decode(ctx).materialize(ctx.arena()); } - throw new VortexException("no decoder registered for " + node.encodingId() - + " (or encoding has no primary segment)"); + String id = switch (node) { + case KnownArrayNode k -> k.encodingId().id(); + case UnknownArrayNode u -> u.rawEncodingId(); + }; + throw new VortexException("no decoder registered for " + id + " (or encoding has no primary segment)"); } private static UnknownArray decodeUnknown(DecodeContext ctx, ArrayNode node) { - String rawId = node.encodingId(); + String rawId = switch (node) { + case KnownArrayNode k -> k.encodingId().id(); + case UnknownArrayNode u -> u.rawEncodingId(); + }; MemorySegment[] bufs = new MemorySegment[node.bufferIndices().length]; for (int i = 0; i < bufs.length; i++) { bufs[i] = ctx.buffer(i); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ArrayNode.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ArrayNode.java index cbae13a8..15cd857c 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ArrayNode.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ArrayNode.java @@ -4,38 +4,34 @@ import java.lang.foreign.MemorySegment; -/// Encoded array node as stored in a Flat layout segment — the in-file representation before -/// decoding. +/// Encoded array node as stored in a Flat layout segment. +/// In-file representation before decoding; mirrors the Go ArrayNode struct. /// -/// The encoding id is carried as the raw string from the wire, because encoding ids ARE strings in -/// the format (`"vortex.flat"`, `"fastlanes.bitpacked"`, …). Whether an id is decodable is the -/// [io.github.dfa1.vortex.reader.ReadRegistry]'s question at decode time, not a property of the -/// node: an id no registered decoder claims either decodes as a passthrough -/// [io.github.dfa1.vortex.reader.array.UnknownArray] (when -/// [io.github.dfa1.vortex.reader.ReadRegistry#isAllowUnknown()] is set) or fails the decode. -/// -/// @param encodingId the raw encoding id string from the file -/// @param metadata encoding-specific metadata bytes, or `null` -/// @param children child nodes -/// @param bufferIndices segment buffer indices for this node -@SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. -public record ArrayNode( - String encodingId, - MemorySegment metadata, - ArrayNode[] children, - int[] bufferIndices -) { +/// Sealed: a node is either [KnownArrayNode] (id resolves to an [EncodingId]) or +/// [UnknownArrayNode] (id is an arbitrary string only meaningful for +/// [io.github.dfa1.vortex.reader.ReadRegistry#isAllowUnknown()] passthrough decode). +public sealed interface ArrayNode permits KnownArrayNode, UnknownArrayNode { - /// Short factory for the common case: a node whose encoding id is well-known. Mostly used by - /// tests and helper code that converts an `EncodeNode` tree back into an `ArrayNode` tree. + /// Short factory for the common case: a node whose encoding id is well-known. + /// Mostly used by tests and helper code that converts an `EncodeNode` tree back into + /// an `ArrayNode` tree. /// /// @param encodingId the well-known encoding identifier /// @param metadata encoding-specific metadata bytes, or `null` /// @param children child nodes /// @param bufferIndices segment buffer indices for this node - /// @return an [ArrayNode] carrying the id's wire string - public static ArrayNode of(EncodingId encodingId, MemorySegment metadata, ArrayNode[] children, + /// @return a [KnownArrayNode] with the given fields + static ArrayNode of(EncodingId encodingId, MemorySegment metadata, ArrayNode[] children, int[] bufferIndices) { - return new ArrayNode(encodingId.id(), metadata, children, bufferIndices); + return new KnownArrayNode(encodingId, metadata, children, bufferIndices); } + + /// @return encoding-specific metadata bytes, or `null` + MemorySegment metadata(); + + /// @return child nodes + ArrayNode[] children(); + + /// @return segment buffer indices for this node + int[] bufferIndices(); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/KnownArrayNode.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/KnownArrayNode.java new file mode 100644 index 00000000..bcc5f58d --- /dev/null +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/KnownArrayNode.java @@ -0,0 +1,20 @@ +package io.github.dfa1.vortex.reader.decode; + +import io.github.dfa1.vortex.core.model.EncodingId; + +import java.lang.foreign.MemorySegment; + +/// Array node whose encoding id is well-known to this build (an [EncodingId] enum constant). +/// +/// @param encodingId well-known encoding id +/// @param metadata encoding-specific metadata bytes, or `null` +/// @param children child nodes +/// @param bufferIndices segment buffer indices +@SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. +public record KnownArrayNode( + EncodingId encodingId, + MemorySegment metadata, + ArrayNode[] children, + int[] bufferIndices +) implements ArrayNode { +} diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/UnknownArrayNode.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/UnknownArrayNode.java new file mode 100644 index 00000000..2804d04c --- /dev/null +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/UnknownArrayNode.java @@ -0,0 +1,21 @@ +package io.github.dfa1.vortex.reader.decode; + +import java.lang.foreign.MemorySegment; + +/// Array node whose encoding id is not a recognized [io.github.dfa1.vortex.core.model.EncodingId]. +/// Produced when a file uses an encoding this build does not know about. Decoded as +/// [io.github.dfa1.vortex.reader.array.UnknownArray] when +/// [io.github.dfa1.vortex.reader.ReadRegistry#isAllowUnknown()] is set; otherwise the decode call throws. +/// +/// @param rawEncodingId the raw encoding id string from the file +/// @param metadata encoding-specific metadata bytes, or `null` +/// @param children child nodes +/// @param bufferIndices segment buffer indices +@SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. +public record UnknownArrayNode( + String rawEncodingId, + MemorySegment metadata, + ArrayNode[] children, + int[] bufferIndices +) implements ArrayNode { +} diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/FlatSegmentDecoderDecodeTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/FlatSegmentDecoderDecodeTest.java index 039920c2..1aa40de1 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/FlatSegmentDecoderDecodeTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/FlatSegmentDecoderDecodeTest.java @@ -17,11 +17,11 @@ /// Successful flat-segment decode path — complements [FlatSegmentBoundsSecurityTest] (which only /// drives the rejection paths). A buffer descriptor with non-zero padding exercises the offset -/// walk, and an unknown encoding id exercises the unknown-id passthrough through an +/// walk, and an unknown encoding id exercises the `UnknownArrayNode` fallback through an /// allow-unknown registry. Together these pin two otherwise-untested spots: /// - the `dataOffset += padding` accumulation: with padding > 0, flipping `+=` to `-=` slices at a /// negative offset and fails, so a clean decode proves the addition. -/// - the unknown-id node construction: mishandling an unresolvable id there yields a +/// - the `orElseGet(() -> new UnknownArrayNode(...))` fallback: returning `null` there yields a /// null node and the decode would not produce an `UnknownArray`. class FlatSegmentDecoderDecodeTest { @@ -46,7 +46,7 @@ void decode_unknownEncodingWithBufferPadding_returnsUnknownArray() { DType.I32, 0, arena); // Then — the allow-unknown path produced an UnknownArray (proves both the +padding - // walk and the unknown-id passthrough ran) + // walk and the UnknownArrayNode fallback ran) assertThat(result).isInstanceOf(UnknownArray.class); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ReadRegistryTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ReadRegistryTest.java index e0f0e7ad..f82b91cd 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/ReadRegistryTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ReadRegistryTest.java @@ -8,6 +8,7 @@ import io.github.dfa1.vortex.reader.decode.ArrayNode; import io.github.dfa1.vortex.reader.decode.DecodeContext; import io.github.dfa1.vortex.reader.decode.EncodingDecoder; +import io.github.dfa1.vortex.reader.decode.UnknownArrayNode; import org.junit.jupiter.api.Test; import java.lang.foreign.Arena; @@ -24,7 +25,7 @@ class ReadRegistryTest { void decodeUnknownEncodingThrowsByDefault() { // Given ReadRegistry sut = ReadRegistry.empty(); - ArrayNode node = new ArrayNode("some.unknown", + ArrayNode node = new UnknownArrayNode("some.unknown", MemorySegment.ofArray(new byte[0]), new ArrayNode[0], new int[0]); DecodeContext ctx = new DecodeContext(node, DTypes.I32, 0L, new MemorySegment[0], sut, Arena.ofAuto()); @@ -74,7 +75,7 @@ void decodeUnknownEncodingReturnsUnknownArrayWhenAllowed() { MemorySegment metadata = MemorySegment.ofArray(new byte[]{1, 2, 3}); MemorySegment buf = Arena.ofAuto().allocate(4); buf.set(java.lang.foreign.ValueLayout.JAVA_INT, 0, 42); - ArrayNode node = new ArrayNode("some.unknown", + ArrayNode node = new UnknownArrayNode("some.unknown", metadata, new ArrayNode[0], new int[]{0}); DecodeContext ctx = new DecodeContext(node, DTypes.I32, 5L, new MemorySegment[]{buf}, sut, Arena.ofAuto()); @@ -102,7 +103,7 @@ void decodeUnknownEncodingWrapsChildrenAsUnknown() { // its parent is unknown — mirrors Rust decode_foreign in vortex-array/src/serde.rs:380. ArrayNode child = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, MemorySegment.ofArray(new byte[0]), new ArrayNode[0], new int[0]); - ArrayNode parent = new ArrayNode("some.unknown", + ArrayNode parent = new UnknownArrayNode("some.unknown", MemorySegment.ofArray(new byte[0]), new ArrayNode[]{child}, new int[0]); DecodeContext ctx = new DecodeContext(parent, DTypes.I32, 0L, new MemorySegment[0], sut, Arena.ofAuto()); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoderTest.java index f9fa0d9b..60eeff12 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoderTest.java @@ -9,6 +9,7 @@ import io.github.dfa1.vortex.reader.decode.DecodeContext; import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.reader.decode.KnownArrayNode; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; import io.github.dfa1.vortex.core.proto.ProtoRLEMetadata; @@ -33,12 +34,12 @@ class RleEncodingEncoderTest { private static final RleEncodingDecoder DECODER = new RleEncodingDecoder(); private static final ReadRegistry REGISTRY = TestRegistry.ofDecoders(DECODER, new PrimitiveEncodingDecoder()); - private static ArrayNode toArrayNode(EncodeNode enc) { + private static KnownArrayNode toArrayNode(EncodeNode enc) { ArrayNode[] children = new ArrayNode[enc.children().length]; for (int i = 0; i < children.length; i++) { children[i] = toArrayNode(enc.children()[i]); } - return ArrayNode.of(enc.encodingId(), enc.metadata(), children, enc.bufferIndices()); + return new KnownArrayNode(enc.encodingId(), enc.metadata(), children, enc.bufferIndices()); } @Nested @@ -267,14 +268,14 @@ void decode_nullableIndices_returnsMaskedArrayWithCorrectValidity() { originalBufs.add(validityBuf); MemorySegment[] segments = originalBufs.toArray(new MemorySegment[0]); - ArrayNode origRoot = toArrayNode(encoded.rootNode()); - ArrayNode origIndices = origRoot.children()[1]; + KnownArrayNode origRoot = toArrayNode(encoded.rootNode()); + KnownArrayNode origIndices = (KnownArrayNode) origRoot.children()[1]; ArrayNode validityNode = ArrayNode.of( EncodingId.VORTEX_BOOL, null, new ArrayNode[0], new int[]{3}); - ArrayNode nullableIndices = new ArrayNode( + ArrayNode nullableIndices = ArrayNode.of( origIndices.encodingId(), origIndices.metadata(), new ArrayNode[]{validityNode}, origIndices.bufferIndices()); - ArrayNode root = new ArrayNode( + ArrayNode root = ArrayNode.of( origRoot.encodingId(), origRoot.metadata(), new ArrayNode[]{origRoot.children()[0], nullableIndices, origRoot.children()[2]}, origRoot.bufferIndices()); From f2cbd22ded4f52caa60d4d2349e63a068fceb9fe Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 4 Jul 2026 09:07:34 +0200 Subject: [PATCH 3/7] docs(core): drop stale not-yet-implemented on ids MaskedEncodingDecoder and PatchedEncodingDecoder are implemented and ServiceLoader-registered; the enum constants still claimed otherwise. Co-Authored-By: Claude Fable 5 --- .../java/io/github/dfa1/vortex/core/model/EncodingId.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/github/dfa1/vortex/core/model/EncodingId.java b/core/src/main/java/io/github/dfa1/vortex/core/model/EncodingId.java index d56c8f33..5267d3ce 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/model/EncodingId.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/model/EncodingId.java @@ -72,10 +72,9 @@ public enum EncodingId { /// FastLanes run-length encoding (`fastlanes.rle`). FASTLANES_RLE("fastlanes.rle"), - // Known in Rust but not yet implemented; registered so EncodingId.parse() resolves - /// Masked encoding (not yet implemented; registered to prevent parse errors). + /// Masked encoding (`vortex.masked`): payload child plus optional validity bitmap child. VORTEX_MASKED("vortex.masked"), - /// Patched encoding (not yet implemented; registered to prevent parse errors). + /// Patched encoding (`vortex.patched`): base child with sparse positional patch overrides. VORTEX_PATCHED("vortex.patched"), /// Variant logical encoding: canonical container over `core_storage` plus an optional shredded child. VORTEX_VARIANT("vortex.variant"), From ea88a91bfc2b53d5a862d5a6ca650abb96863155 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 4 Jul 2026 10:50:13 +0200 Subject: [PATCH 4/7] refactor(core): EncodingId becomes a sealed interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sealed interface EncodingId permits WellKnown, Custom: the spec constants move verbatim into the nested WellKnown enum and every wire string now has a typed representation — parse() is total, returning the WellKnown match or a Custom wrapping the raw id. Custom's compact constructor rejects null, blank, and ids that collide with a WellKnown wire string, so the two variants never alias as map keys. All constants are re-exported as WellKnown-typed interface fields, keeping the existing EncodingId.VORTEX_FOO call sites source compatible. KnownArrayNode narrows its component to WellKnown (a known node can no longer hold a custom id); ArrayNode.of dispatches WellKnown/Custom to Known/UnknownArrayNode. EncodingDecoder and EncodingEncoder keep the interface in their signatures, which for the first time lets third-party codecs declare ids outside the spec set. EncodingId extends Serializable to preserve the enum's implicit serializability for the VortexException field. Co-Authored-By: Claude Fable 5 --- .../dfa1/vortex/core/model/EncodingId.java | 318 ++++++++++++------ .../vortex/core/model/EncodingIdTest.java | 58 +++- .../vortex/reader/FlatSegmentDecoder.java | 9 +- .../dfa1/vortex/reader/ReadRegistry.java | 25 +- .../dfa1/vortex/reader/decode/ArrayNode.java | 16 +- .../vortex/reader/decode/KnownArrayNode.java | 4 +- .../dfa1/vortex/reader/VortexReaderTest.java | 2 +- .../writer/encode/RleEncodingEncoderTest.java | 5 +- 8 files changed, 302 insertions(+), 135 deletions(-) diff --git a/core/src/main/java/io/github/dfa1/vortex/core/model/EncodingId.java b/core/src/main/java/io/github/dfa1/vortex/core/model/EncodingId.java index 5267d3ce..6aa41458 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/model/EncodingId.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/model/EncodingId.java @@ -1,113 +1,235 @@ package io.github.dfa1.vortex.core.model; +import java.io.Serializable; import java.util.Map; -import java.util.Optional; +import java.util.Objects; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; -/// Strongly-typed encoding identifier used in place of raw strings. -public enum EncodingId { - /// Canonical flat primitive encoding (`vortex.primitive`). - VORTEX_PRIMITIVE("vortex.primitive"), - /// Bit-packed boolean encoding (`vortex.bool`). - VORTEX_BOOL("vortex.bool"), - /// Dictionary encoding for low-cardinality columns (`vortex.dict`). - VORTEX_DICT("vortex.dict"), - /// Sparse encoding for columns with many nulls or zeros (`vortex.sparse`). - VORTEX_SPARSE("vortex.sparse"), - /// Sequence encoding (`vortex.sequence`). - VORTEX_SEQUENCE("vortex.sequence"), - /// Run-end encoding for sorted/repetitive columns (`vortex.runend`). - VORTEX_RUNEND("vortex.runend"), - /// Constant encoding — all elements share one value (`vortex.constant`). - VORTEX_CONSTANT("vortex.constant"), - /// ALP (Adaptive Lossless floating-Point) encoding for F32/F64 (`vortex.alp`). - VORTEX_ALP("vortex.alp"), - /// Variable-length binary encoding (`vortex.varbin`). - VORTEX_VARBIN("vortex.varbin"), - /// FSST compressed string encoding (`vortex.fsst`). - VORTEX_FSST("vortex.fsst"), - /// All-null encoding (`vortex.null`). - VORTEX_NULL("vortex.null"), - /// One-byte-per-boolean encoding (`vortex.bytebool`). - VORTEX_BYTEBOOL("vortex.bytebool"), - /// Zig-zag encoding for signed integers (`vortex.zigzag`). - VORTEX_ZIGZAG("vortex.zigzag"), - /// Extension type wrapper encoding (`vortex.ext`). - VORTEX_EXT("vortex.ext"), - /// Variable-length binary view encoding (`vortex.varbinview`). - VORTEX_VARBINVIEW("vortex.varbinview"), - /// pcodec (Pco) floating-point/integer encoding (`vortex.pco`). - VORTEX_PCO("vortex.pco"), - /// Canonical flat decimal storage (`vortex.decimal`). - VORTEX_DECIMAL("vortex.decimal"), - /// Decimal split into MSP + LSP children (`vortex.decimal_byte_parts`). - VORTEX_DECIMAL_BYTE_PARTS("vortex.decimal_byte_parts"), - /// Timestamp split into days/seconds/subseconds (`vortex.datetimeparts`). - VORTEX_DATETIMEPARTS("vortex.datetimeparts"), - /// Zstandard compressed encoding (`vortex.zstd`). - VORTEX_ZSTD("vortex.zstd"), - /// Fixed-size list encoding (`vortex.fixed_size_list`). - VORTEX_FIXED_SIZE_LIST("vortex.fixed_size_list"), - /// Variable-length list encoding (`vortex.list`). - VORTEX_LIST("vortex.list"), - /// List-view encoding (`vortex.listview`). - VORTEX_LISTVIEW("vortex.listview"), - /// ALP-RD (ALP with remainder dictionary) encoding (`vortex.alprd`). - VORTEX_ALPRD("vortex.alprd"), - - // Layout encoding IDs included so parser/registry can represent them safely - /// Chunked layout encoding (`vortex.chunked`). - VORTEX_CHUNKED("vortex.chunked"), - /// Struct layout encoding (`vortex.struct`). - VORTEX_STRUCT("vortex.struct"), - - /// FastLanes bit-packed encoding (`fastlanes.bitpacked`). - FASTLANES_BITPACKED("fastlanes.bitpacked"), - /// FastLanes frame-of-reference encoding (`fastlanes.for`). - FASTLANES_FOR("fastlanes.for"), - /// FastLanes delta encoding (`fastlanes.delta`). - FASTLANES_DELTA("fastlanes.delta"), - /// FastLanes run-length encoding (`fastlanes.rle`). - FASTLANES_RLE("fastlanes.rle"), - - /// Masked encoding (`vortex.masked`): payload child plus optional validity bitmap child. - VORTEX_MASKED("vortex.masked"), - /// Patched encoding (`vortex.patched`): base child with sparse positional patch overrides. - VORTEX_PATCHED("vortex.patched"), - /// Variant logical encoding: canonical container over `core_storage` plus an optional shredded child. - VORTEX_VARIANT("vortex.variant"), - ; - - // O(1) access to EncodingId by its string representation - private static final Map LOOKUP = Stream.of(EncodingId.values()) - .collect(Collectors.toUnmodifiableMap(EncodingId::id, Function.identity())); - private final String id; - - EncodingId(String id) { - this.id = id; - } +/// Identity of an array encoding — either a spec [WellKnown] constant or a third-party [Custom] id. +/// +/// The wire representation is always a string (e.g. `"vortex.flat"`); [#parse(String)] maps any +/// such string to a typed value, and [#id()] recovers the wire string from a typed value. +/// +/// Extends [Serializable] so a [Custom] or [WellKnown] carried on a +/// [io.github.dfa1.vortex.core.error.VortexException] survives serialization, matching the prior +/// enum's implicit serializability. +public sealed interface EncodingId extends Serializable permits EncodingId.WellKnown, EncodingId.Custom { - /// Parses a raw encoding id string into the matching constant. - /// Used by `ReadRegistry` to discriminate `KnownArrayNode` from `UnknownArrayNode`; - /// callers that demand a known id chain `.orElseThrow(...)`. + /// Returns the wire string of this encoding id (e.g. `"vortex.flat"`). /// - /// @param id raw encoding id string (e.g. `"vortex.primitive"`) - /// @return matching constant, or empty if not recognized - public static Optional parse(String id) { - return Optional.ofNullable(LOOKUP.get(id)); - } + /// @return the wire string of this encoding id + String id(); - /// Returns the raw encoding id string for this constant (e.g. `"vortex.primitive"`). + /// Total parse: every non-null string has a typed representation. + /// Returns the matching [WellKnown] constant, else a [Custom] wrapping the raw string. /// - /// @return the raw string encoding id - public String id() { - return id; + /// @param raw the raw encoding id string (e.g. `"vortex.primitive"`) + /// @return the matching [WellKnown] constant, or a [Custom] wrapping `raw` if none matches + static EncodingId parse(String raw) { + WellKnown known = WellKnown.byId(raw); + return known != null ? known : new Custom(raw); } - @Override - public String toString() { - return id; + /// Encoding ids defined by the Vortex specification and understood by this build. + enum WellKnown implements EncodingId { + /// Canonical flat primitive encoding (`vortex.primitive`). + VORTEX_PRIMITIVE("vortex.primitive"), + /// Bit-packed boolean encoding (`vortex.bool`). + VORTEX_BOOL("vortex.bool"), + /// Dictionary encoding for low-cardinality columns (`vortex.dict`). + VORTEX_DICT("vortex.dict"), + /// Sparse encoding for columns with many nulls or zeros (`vortex.sparse`). + VORTEX_SPARSE("vortex.sparse"), + /// Sequence encoding (`vortex.sequence`). + VORTEX_SEQUENCE("vortex.sequence"), + /// Run-end encoding for sorted/repetitive columns (`vortex.runend`). + VORTEX_RUNEND("vortex.runend"), + /// Constant encoding — all elements share one value (`vortex.constant`). + VORTEX_CONSTANT("vortex.constant"), + /// ALP (Adaptive Lossless floating-Point) encoding for F32/F64 (`vortex.alp`). + VORTEX_ALP("vortex.alp"), + /// Variable-length binary encoding (`vortex.varbin`). + VORTEX_VARBIN("vortex.varbin"), + /// FSST compressed string encoding (`vortex.fsst`). + VORTEX_FSST("vortex.fsst"), + /// All-null encoding (`vortex.null`). + VORTEX_NULL("vortex.null"), + /// One-byte-per-boolean encoding (`vortex.bytebool`). + VORTEX_BYTEBOOL("vortex.bytebool"), + /// Zig-zag encoding for signed integers (`vortex.zigzag`). + VORTEX_ZIGZAG("vortex.zigzag"), + /// Extension type wrapper encoding (`vortex.ext`). + VORTEX_EXT("vortex.ext"), + /// Variable-length binary view encoding (`vortex.varbinview`). + VORTEX_VARBINVIEW("vortex.varbinview"), + /// pcodec (Pco) floating-point/integer encoding (`vortex.pco`). + VORTEX_PCO("vortex.pco"), + /// Canonical flat decimal storage (`vortex.decimal`). + VORTEX_DECIMAL("vortex.decimal"), + /// Decimal split into MSP + LSP children (`vortex.decimal_byte_parts`). + VORTEX_DECIMAL_BYTE_PARTS("vortex.decimal_byte_parts"), + /// Timestamp split into days/seconds/subseconds (`vortex.datetimeparts`). + VORTEX_DATETIMEPARTS("vortex.datetimeparts"), + /// Zstandard compressed encoding (`vortex.zstd`). + VORTEX_ZSTD("vortex.zstd"), + /// Fixed-size list encoding (`vortex.fixed_size_list`). + VORTEX_FIXED_SIZE_LIST("vortex.fixed_size_list"), + /// Variable-length list encoding (`vortex.list`). + VORTEX_LIST("vortex.list"), + /// List-view encoding (`vortex.listview`). + VORTEX_LISTVIEW("vortex.listview"), + /// ALP-RD (ALP with remainder dictionary) encoding (`vortex.alprd`). + VORTEX_ALPRD("vortex.alprd"), + + // Layout encoding IDs included so parser/registry can represent them safely + /// Chunked layout encoding (`vortex.chunked`). + VORTEX_CHUNKED("vortex.chunked"), + /// Struct layout encoding (`vortex.struct`). + VORTEX_STRUCT("vortex.struct"), + + /// FastLanes bit-packed encoding (`fastlanes.bitpacked`). + FASTLANES_BITPACKED("fastlanes.bitpacked"), + /// FastLanes frame-of-reference encoding (`fastlanes.for`). + FASTLANES_FOR("fastlanes.for"), + /// FastLanes delta encoding (`fastlanes.delta`). + FASTLANES_DELTA("fastlanes.delta"), + /// FastLanes run-length encoding (`fastlanes.rle`). + FASTLANES_RLE("fastlanes.rle"), + + /// Masked encoding (`vortex.masked`): payload child plus optional validity bitmap child. + VORTEX_MASKED("vortex.masked"), + /// Patched encoding (`vortex.patched`): base child with sparse positional patch overrides. + VORTEX_PATCHED("vortex.patched"), + /// Variant logical encoding: canonical container over `core_storage` plus an optional shredded child. + VORTEX_VARIANT("vortex.variant"), + ; + + // O(1) access to a WellKnown constant by its string representation + private static final Map LOOKUP = Stream.of(values()) + .collect(Collectors.toUnmodifiableMap(WellKnown::id, Function.identity())); + private final String id; + + WellKnown(String id) { + this.id = id; + } + + /// Returns the well-known constant whose wire string is `id`, or `null` if none matches. + /// + /// @param id the wire string to look up (may be `null`) + /// @return the matching constant, or `null` if unrecognized + static WellKnown byId(String id) { + return LOOKUP.get(id); + } + + @Override + public String id() { + return id; + } + + @Override + public String toString() { + return id; + } + } + + /// A third-party encoding id whose wire string is not part of the [WellKnown] set. + /// + /// @param id the wire string of this encoding id; must be non-blank and must not collide + /// with a [WellKnown] wire string + record Custom(String id) implements EncodingId { + + /// Validates that `id` is a usable custom encoding id. + /// + /// @param id the wire string of this encoding id + /// @throws NullPointerException if `id` is `null` + /// @throws IllegalArgumentException if `id` is blank or matches a [WellKnown] wire string + public Custom { + Objects.requireNonNull(id, "id"); + if (id.isBlank()) { + throw new IllegalArgumentException("encoding id must not be blank"); + } + WellKnown wellKnown = WellKnown.byId(id); + if (wellKnown != null) { + throw new IllegalArgumentException( + "\"" + id + "\" is a well-known encoding id; use EncodingId." + wellKnown.name() + " instead"); + } + } + + @Override + public String toString() { + return id; + } } + + // Re-export every WellKnown constant, typed as WellKnown, so existing `EncodingId.VORTEX_FOO` + // call sites keep compiling and remain usable wherever a WellKnown is required. + + /// Well-known `vortex.primitive` id. + WellKnown VORTEX_PRIMITIVE = WellKnown.VORTEX_PRIMITIVE; + /// Well-known `vortex.bool` id. + WellKnown VORTEX_BOOL = WellKnown.VORTEX_BOOL; + /// Well-known `vortex.dict` id. + WellKnown VORTEX_DICT = WellKnown.VORTEX_DICT; + /// Well-known `vortex.sparse` id. + WellKnown VORTEX_SPARSE = WellKnown.VORTEX_SPARSE; + /// Well-known `vortex.sequence` id. + WellKnown VORTEX_SEQUENCE = WellKnown.VORTEX_SEQUENCE; + /// Well-known `vortex.runend` id. + WellKnown VORTEX_RUNEND = WellKnown.VORTEX_RUNEND; + /// Well-known `vortex.constant` id. + WellKnown VORTEX_CONSTANT = WellKnown.VORTEX_CONSTANT; + /// Well-known `vortex.alp` id. + WellKnown VORTEX_ALP = WellKnown.VORTEX_ALP; + /// Well-known `vortex.varbin` id. + WellKnown VORTEX_VARBIN = WellKnown.VORTEX_VARBIN; + /// Well-known `vortex.fsst` id. + WellKnown VORTEX_FSST = WellKnown.VORTEX_FSST; + /// Well-known `vortex.null` id. + WellKnown VORTEX_NULL = WellKnown.VORTEX_NULL; + /// Well-known `vortex.bytebool` id. + WellKnown VORTEX_BYTEBOOL = WellKnown.VORTEX_BYTEBOOL; + /// Well-known `vortex.zigzag` id. + WellKnown VORTEX_ZIGZAG = WellKnown.VORTEX_ZIGZAG; + /// Well-known `vortex.ext` id. + WellKnown VORTEX_EXT = WellKnown.VORTEX_EXT; + /// Well-known `vortex.varbinview` id. + WellKnown VORTEX_VARBINVIEW = WellKnown.VORTEX_VARBINVIEW; + /// Well-known `vortex.pco` id. + WellKnown VORTEX_PCO = WellKnown.VORTEX_PCO; + /// Well-known `vortex.decimal` id. + WellKnown VORTEX_DECIMAL = WellKnown.VORTEX_DECIMAL; + /// Well-known `vortex.decimal_byte_parts` id. + WellKnown VORTEX_DECIMAL_BYTE_PARTS = WellKnown.VORTEX_DECIMAL_BYTE_PARTS; + /// Well-known `vortex.datetimeparts` id. + WellKnown VORTEX_DATETIMEPARTS = WellKnown.VORTEX_DATETIMEPARTS; + /// Well-known `vortex.zstd` id. + WellKnown VORTEX_ZSTD = WellKnown.VORTEX_ZSTD; + /// Well-known `vortex.fixed_size_list` id. + WellKnown VORTEX_FIXED_SIZE_LIST = WellKnown.VORTEX_FIXED_SIZE_LIST; + /// Well-known `vortex.list` id. + WellKnown VORTEX_LIST = WellKnown.VORTEX_LIST; + /// Well-known `vortex.listview` id. + WellKnown VORTEX_LISTVIEW = WellKnown.VORTEX_LISTVIEW; + /// Well-known `vortex.alprd` id. + WellKnown VORTEX_ALPRD = WellKnown.VORTEX_ALPRD; + /// Well-known `vortex.chunked` id. + WellKnown VORTEX_CHUNKED = WellKnown.VORTEX_CHUNKED; + /// Well-known `vortex.struct` id. + WellKnown VORTEX_STRUCT = WellKnown.VORTEX_STRUCT; + /// Well-known `fastlanes.bitpacked` id. + WellKnown FASTLANES_BITPACKED = WellKnown.FASTLANES_BITPACKED; + /// Well-known `fastlanes.for` id. + WellKnown FASTLANES_FOR = WellKnown.FASTLANES_FOR; + /// Well-known `fastlanes.delta` id. + WellKnown FASTLANES_DELTA = WellKnown.FASTLANES_DELTA; + /// Well-known `fastlanes.rle` id. + WellKnown FASTLANES_RLE = WellKnown.FASTLANES_RLE; + /// Well-known `vortex.masked` id. + WellKnown VORTEX_MASKED = WellKnown.VORTEX_MASKED; + /// Well-known `vortex.patched` id. + WellKnown VORTEX_PATCHED = WellKnown.VORTEX_PATCHED; + /// Well-known `vortex.variant` id. + WellKnown VORTEX_VARIANT = WellKnown.VORTEX_VARIANT; } diff --git a/core/src/test/java/io/github/dfa1/vortex/core/model/EncodingIdTest.java b/core/src/test/java/io/github/dfa1/vortex/core/model/EncodingIdTest.java index 43baebf3..9cb0b2a6 100644 --- a/core/src/test/java/io/github/dfa1/vortex/core/model/EncodingIdTest.java +++ b/core/src/test/java/io/github/dfa1/vortex/core/model/EncodingIdTest.java @@ -6,6 +6,7 @@ import org.junit.jupiter.params.provider.EnumSource; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; class EncodingIdTest { @@ -13,16 +14,51 @@ class EncodingIdTest { class Parse { @ParameterizedTest - @EnumSource(EncodingId.class) - void parse_knownId_returnsMatchingConstant(EncodingId id) { - // Given / When / Then — every declared constant round-trips through its wire id - assertThat(EncodingId.parse(id.id())).contains(id); + @EnumSource(EncodingId.WellKnown.class) + void parse_knownId_returnsMatchingConstant(EncodingId.WellKnown id) { + // Given the wire string of a well-known constant + // When + EncodingId result = EncodingId.parse(id.id()); + // Then the same constant comes back + assertThat(result).isSameAs(id); } @Test - void parse_unknownId_returnsEmpty() { - // Given / When / Then — non-throwing miss so the registry can route to passthrough - assertThat(EncodingId.parse("supermario")).isEmpty(); + void parse_unknownId_returnsCustomWrappingRawId() { + // Given a wire string no build knows about + String raw = "supermario"; + // When — parse is total, so a miss is a typed Custom rather than an empty Optional + EncodingId result = EncodingId.parse(raw); + // Then + assertThat(result).isEqualTo(new EncodingId.Custom(raw)); + assertThat(result.id()).isEqualTo(raw); + } + } + + @Nested + class CustomInvariants { + + @Test + void construct_nullId_throwsNullPointerException() { + // Given / When / Then — a Custom must always carry a wire string + assertThatThrownBy(() -> new EncodingId.Custom(null)) + .isInstanceOf(NullPointerException.class); + } + + @Test + void construct_blankId_throwsIllegalArgumentException() { + // Given / When / Then — blank ids have no wire representation + assertThatThrownBy(() -> new EncodingId.Custom(" ")) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void construct_wellKnownId_throwsIllegalArgumentException() { + // Given a wire string that already names a well-known constant + // When / Then — Custom refuses to shadow it and points at the constant to use instead + assertThatThrownBy(() -> new EncodingId.Custom("vortex.primitive")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("VORTEX_PRIMITIVE"); } } @@ -30,15 +66,15 @@ void parse_unknownId_returnsEmpty() { class Properties { @ParameterizedTest - @EnumSource(EncodingId.class) - void id_isNonBlankString(EncodingId id) { + @EnumSource(EncodingId.WellKnown.class) + void id_isNonBlankString(EncodingId.WellKnown id) { // Given / When / Then assertThat(id.id()).isNotBlank(); } @ParameterizedTest - @EnumSource(EncodingId.class) - void toString_equalsId(EncodingId id) { + @EnumSource(EncodingId.WellKnown.class) + void toString_equalsId(EncodingId.WellKnown id) { // Given / When / Then assertThat(id).hasToString(id.id()); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/FlatSegmentDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/FlatSegmentDecoder.java index 6fe9d8cd..5ec741dd 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/FlatSegmentDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/FlatSegmentDecoder.java @@ -99,8 +99,11 @@ private static ArrayNode convertArrayNode( } MemorySegment meta = fbs.metadataAsSegment(); - return EncodingId.parse(rawEncodingId) - .map(known -> new KnownArrayNode(known, meta, children, bufferIndices)) - .orElseGet(() -> new UnknownArrayNode(rawEncodingId, meta, children, bufferIndices)); + return switch (EncodingId.parse(rawEncodingId)) { + case EncodingId.WellKnown wellKnown -> + new KnownArrayNode(wellKnown, meta, children, bufferIndices); + case EncodingId.Custom custom -> + new UnknownArrayNode(custom.id(), meta, children, bufferIndices); + }; } } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ReadRegistry.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ReadRegistry.java index 9faa572d..09070597 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ReadRegistry.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ReadRegistry.java @@ -12,7 +12,6 @@ import java.lang.foreign.MemorySegment; import java.util.Collections; -import java.util.Comparator; import java.util.Map; import java.util.ServiceLoader; import java.util.TreeMap; @@ -23,16 +22,14 @@ /// via the [#loadAll()] and [#empty()] convenience factories. public final class ReadRegistry { - private final Map decoders; + private final Map decoders; private final boolean allowUnknown; - private ReadRegistry(Map decoders, boolean allowUnknown) { - // Order by encoding name, mirroring WriteRegistry (Enum.compareTo is final, so a Comparator - // is required to sort by id rather than ordinal). Decode dispatch is keyed, so order is not - // load-bearing here, but a stable order keeps the two registries consistent. - var sorted = new TreeMap(Comparator.comparing(EncodingId::id)); - sorted.putAll(decoders); - this.decoders = Collections.unmodifiableMap(sorted); + private ReadRegistry(Map decoders, boolean allowUnknown) { + // Keyed by the wire string, ordered naturally by it, mirroring WriteRegistry. Decode + // dispatch is keyed, so order is not load-bearing here, but a stable order keeps the two + // registries consistent. + this.decoders = Collections.unmodifiableMap(new TreeMap<>(decoders)); this.allowUnknown = allowUnknown; } @@ -70,7 +67,7 @@ public boolean isAllowUnknown() { /// @param encodingId the encoding id to query /// @return `true` if a decoder is registered public boolean hasDecoder(EncodingId encodingId) { - return decoders.containsKey(encodingId); + return decoders.containsKey(encodingId.id()); } /// Decodes the array described by `ctx`. @@ -80,7 +77,7 @@ public boolean hasDecoder(EncodingId encodingId) { public Array decode(DecodeContext ctx) { ArrayNode node = ctx.node(); EncodingDecoder decoder = switch (node) { - case KnownArrayNode k -> decoders.get(k.encodingId()); + case KnownArrayNode k -> decoders.get(k.encodingId().id()); case UnknownArrayNode _ -> null; }; if (decoder != null) { @@ -103,7 +100,7 @@ public Array decode(DecodeContext ctx) { public MemorySegment decodeAsSegment(DecodeContext ctx) { ArrayNode node = ctx.node(); EncodingDecoder decoder = switch (node) { - case KnownArrayNode k -> decoders.get(k.encodingId()); + case KnownArrayNode k -> decoders.get(k.encodingId().id()); case UnknownArrayNode _ -> null; }; if (decoder != null) { @@ -143,7 +140,7 @@ private static UnknownArray decodeUnknown(DecodeContext ctx, ArrayNode node) { /// Not thread-safe. Build once, use everywhere — the produced [ReadRegistry] is immutable. public static final class Builder { - private final Map decoders = new TreeMap<>(); + private final Map decoders = new TreeMap<>(); private boolean allowUnknown = false; private Builder() { @@ -155,7 +152,7 @@ private Builder() { /// @return this builder, for chaining /// @throws VortexException if a decoder for the same id is already registered public Builder register(EncodingDecoder decoder) { - EncodingDecoder old = decoders.put(decoder.encodingId(), decoder); + EncodingDecoder old = decoders.put(decoder.encodingId().id(), decoder); if (old != null) { throw new VortexException("decoder %s already registered".formatted(decoder.encodingId())); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ArrayNode.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ArrayNode.java index 15cd857c..520d6ab1 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ArrayNode.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ArrayNode.java @@ -7,23 +7,29 @@ /// Encoded array node as stored in a Flat layout segment. /// In-file representation before decoding; mirrors the Go ArrayNode struct. /// -/// Sealed: a node is either [KnownArrayNode] (id resolves to an [EncodingId]) or +/// Sealed: a node is either [KnownArrayNode] (id resolves to an [EncodingId.WellKnown]) or /// [UnknownArrayNode] (id is an arbitrary string only meaningful for /// [io.github.dfa1.vortex.reader.ReadRegistry#isAllowUnknown()] passthrough decode). public sealed interface ArrayNode permits KnownArrayNode, UnknownArrayNode { - /// Short factory for the common case: a node whose encoding id is well-known. + /// Factory that builds the right node kind for `encodingId`: a [KnownArrayNode] for a + /// well-known id, or an [UnknownArrayNode] for a [EncodingId.Custom] one. /// Mostly used by tests and helper code that converts an `EncodeNode` tree back into /// an `ArrayNode` tree. /// - /// @param encodingId the well-known encoding identifier + /// @param encodingId the encoding identifier /// @param metadata encoding-specific metadata bytes, or `null` /// @param children child nodes /// @param bufferIndices segment buffer indices for this node - /// @return a [KnownArrayNode] with the given fields + /// @return a [KnownArrayNode] for a well-known id, else an [UnknownArrayNode] static ArrayNode of(EncodingId encodingId, MemorySegment metadata, ArrayNode[] children, int[] bufferIndices) { - return new KnownArrayNode(encodingId, metadata, children, bufferIndices); + return switch (encodingId) { + case EncodingId.WellKnown wellKnown -> + new KnownArrayNode(wellKnown, metadata, children, bufferIndices); + case EncodingId.Custom custom -> + new UnknownArrayNode(custom.id(), metadata, children, bufferIndices); + }; } /// @return encoding-specific metadata bytes, or `null` diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/KnownArrayNode.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/KnownArrayNode.java index bcc5f58d..f0aadcef 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/KnownArrayNode.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/KnownArrayNode.java @@ -4,7 +4,7 @@ import java.lang.foreign.MemorySegment; -/// Array node whose encoding id is well-known to this build (an [EncodingId] enum constant). +/// Array node whose encoding id is well-known to this build (an [EncodingId.WellKnown] constant). /// /// @param encodingId well-known encoding id /// @param metadata encoding-specific metadata bytes, or `null` @@ -12,7 +12,7 @@ /// @param bufferIndices segment buffer indices @SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. public record KnownArrayNode( - EncodingId encodingId, + EncodingId.WellKnown encodingId, MemorySegment metadata, ArrayNode[] children, int[] bufferIndices diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/VortexReaderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/VortexReaderTest.java index cf331e94..427497e0 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/VortexReaderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/VortexReaderTest.java @@ -29,7 +29,7 @@ class VortexReaderTest { private static ReadRegistry buildUniversalStubRegistry() { var b = ReadRegistry.builder(); - for (EncodingId encodingId : EncodingId.values()) { + for (EncodingId encodingId : EncodingId.WellKnown.values()) { b.register(new EncodingDecoder() { @Override public EncodingId encodingId() { diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoderTest.java index 60eeff12..bb3717db 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoderTest.java @@ -39,7 +39,10 @@ private static KnownArrayNode toArrayNode(EncodeNode enc) { for (int i = 0; i < children.length; i++) { children[i] = toArrayNode(enc.children()[i]); } - return new KnownArrayNode(enc.encodingId(), enc.metadata(), children, enc.bufferIndices()); + if (!(enc.encodingId() instanceof EncodingId.WellKnown wellKnown)) { + throw new IllegalStateException("expected well-known encoding id: " + enc.encodingId()); + } + return new KnownArrayNode(wellKnown, enc.metadata(), children, enc.bufferIndices()); } @Nested From 21810d7e88760544499748a986c9693cb39fb9a3 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 4 Jul 2026 11:06:37 +0200 Subject: [PATCH 5/7] refactor(reader): ArrayNode carries the typed EncodingId MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Collapse the KnownArrayNode/UnknownArrayNode sealed pair into a single ArrayNode record: with EncodingId sealed, the Known/Unknown bit lives in the id type (WellKnown vs Custom), so the parallel node hierarchy and its switch dances at every dispatch site encode nothing. ReadRegistry returns to the single string-keyed map hit; the allowUnknown passthrough and error messages are unchanged for both former variants. The redundant ArrayNode.of factory is gone — the canonical constructor takes the typed id. Hardening from review: a crafted file with a blank encoding id in the spec table now fails as VortexException in FlatSegmentDecoder (EncodingId.parse rejects blank with IllegalArgumentException, which must not escape the untrusted read path — previously "" flowed to the unknown-id path and could pass through allowUnknown); WriteRegistry's builder TreeMap orders by wire string instead of natural ordering, which a Custom-keyed registration would break. Co-Authored-By: Claude Fable 5 --- .../dfa1/vortex/core/model/EncodingId.java | 8 ++- .../vortex/core/model/EncodingIdTest.java | 10 ++++ .../vortex/reader/FlatSegmentDecoder.java | 14 ++--- .../dfa1/vortex/reader/ReadRegistry.java | 30 ++-------- .../dfa1/vortex/reader/decode/ArrayNode.java | 56 +++++++------------ .../vortex/reader/decode/KnownArrayNode.java | 20 ------- .../reader/decode/UnknownArrayNode.java | 21 ------- .../reader/FlatSegmentDecoderDecodeTest.java | 35 ++++++++++-- .../dfa1/vortex/reader/ReadRegistryTest.java | 13 ++--- .../reader/decode/AlpEncodingDecoderTest.java | 30 +++++----- .../decode/ByteBoolEncodingDecoderTest.java | 2 +- .../DateTimePartsEncodingDecoderTest.java | 10 ++-- .../decode/DecimalEncodingDecoderTest.java | 2 +- .../decode/DeltaEncodingDecoderTest.java | 14 ++--- .../decode/DictEncodingDecoderTest.java | 28 +++++----- .../decode/NullEncodingDecoderTest.java | 2 +- .../decode/PatchedEncodingDecoderTest.java | 14 ++--- .../reader/decode/PcoEncodingDecoderTest.java | 6 +- .../decode/VarBinEncodingDecoderTest.java | 4 +- .../decode/VarBinViewEncodingDecoderTest.java | 6 +- .../decode/VariantEncodingDecoderTest.java | 14 ++--- .../decode/ZigZagEncodingDecoderTest.java | 6 +- .../dfa1/vortex/writer/WriteRegistry.java | 4 +- .../writer/encode/AlpEncodingEncoderTest.java | 12 ++-- ...BitpackedConstantPatchesBroadcastTest.java | 6 +- .../encode/BitpackedEncodingPatchesTest.java | 12 ++-- .../encode/ChunkedEncodingEncoderTest.java | 8 +-- .../DateTimePartsEncodingEncoderTest.java | 2 +- .../writer/encode/DecodeTestHelper.java | 2 +- .../writer/encode/ExtEncodingEncoderTest.java | 6 +- .../FixedSizeListEncodingEncoderTest.java | 4 +- .../FrameOfReferenceEncodingEncoderTest.java | 10 ++-- .../encode/FsstEncodingEncoderTest.java | 10 ++-- .../encode/ListEncodingEncoderTest.java | 8 +-- .../encode/ListViewEncodingEncoderTest.java | 8 +-- .../encode/NullEncodingEncoderTest.java | 2 +- .../encode/PatchedEncodingEncoderTest.java | 4 +- .../encode/PrimitiveEncodingEncoderTest.java | 6 +- .../writer/encode/RleEncodingEncoderTest.java | 18 +++--- .../encode/RunEndEncodingEncoderTest.java | 6 +- .../encode/SequenceEncodingEncoderTest.java | 6 +- .../encode/SparseEncodingEncoderTest.java | 26 ++++----- .../encode/StructEncodingEncoderTest.java | 10 ++-- .../encode/VarBinEncodingEncoderTest.java | 2 +- .../encode/VarBinViewEncodingEncoderTest.java | 4 +- .../encode/VariantEncodingEncoderTest.java | 2 +- .../encode/ZigZagEncodingEncoderTest.java | 4 +- .../encode/ZstdEncodingEncoderTest.java | 14 ++--- 48 files changed, 252 insertions(+), 289 deletions(-) delete mode 100644 reader/src/main/java/io/github/dfa1/vortex/reader/decode/KnownArrayNode.java delete mode 100644 reader/src/main/java/io/github/dfa1/vortex/reader/decode/UnknownArrayNode.java diff --git a/core/src/main/java/io/github/dfa1/vortex/core/model/EncodingId.java b/core/src/main/java/io/github/dfa1/vortex/core/model/EncodingId.java index 6aa41458..6f5c1d90 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/model/EncodingId.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/model/EncodingId.java @@ -22,11 +22,15 @@ public sealed interface EncodingId extends Serializable permits EncodingId.WellK /// @return the wire string of this encoding id String id(); - /// Total parse: every non-null string has a typed representation. - /// Returns the matching [WellKnown] constant, else a [Custom] wrapping the raw string. + /// Parses a wire string into its typed representation: the matching [WellKnown] constant, + /// else a [Custom] wrapping the raw string. Total over every non-blank string; blank input + /// is not a valid encoding id and is rejected by the [Custom] constructor — callers parsing + /// untrusted input must guard blank ids and raise their own domain error. /// /// @param raw the raw encoding id string (e.g. `"vortex.primitive"`) /// @return the matching [WellKnown] constant, or a [Custom] wrapping `raw` if none matches + /// @throws NullPointerException if `raw` is `null` + /// @throws IllegalArgumentException if `raw` is blank static EncodingId parse(String raw) { WellKnown known = WellKnown.byId(raw); return known != null ? known : new Custom(raw); diff --git a/core/src/test/java/io/github/dfa1/vortex/core/model/EncodingIdTest.java b/core/src/test/java/io/github/dfa1/vortex/core/model/EncodingIdTest.java index 9cb0b2a6..e2cfa09f 100644 --- a/core/src/test/java/io/github/dfa1/vortex/core/model/EncodingIdTest.java +++ b/core/src/test/java/io/github/dfa1/vortex/core/model/EncodingIdTest.java @@ -4,6 +4,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -33,6 +34,15 @@ void parse_unknownId_returnsCustomWrappingRawId() { assertThat(result).isEqualTo(new EncodingId.Custom(raw)); assertThat(result.id()).isEqualTo(raw); } + + @ParameterizedTest + @ValueSource(strings = {"", " ", " "}) + void parse_blankId_throwsIllegalArgumentException(String blank) { + // Given / When / Then — blank is not a valid id; parse must not silently wrap it, + // so untrusted-input callers are forced to guard it into their own domain error + assertThatThrownBy(() -> EncodingId.parse(blank)) + .isInstanceOf(IllegalArgumentException.class); + } } @Nested diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/FlatSegmentDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/FlatSegmentDecoder.java index 5ec741dd..bbcb2562 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/FlatSegmentDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/FlatSegmentDecoder.java @@ -10,8 +10,6 @@ import io.github.dfa1.vortex.core.fbs.FbsBuffer; import io.github.dfa1.vortex.reader.decode.ArrayNode; import io.github.dfa1.vortex.reader.decode.DecodeContext; -import io.github.dfa1.vortex.reader.decode.KnownArrayNode; -import io.github.dfa1.vortex.reader.decode.UnknownArrayNode; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -87,6 +85,11 @@ private static ArrayNode convertArrayNode( "array tree depth exceeds limit (" + MAX_ARRAY_TREE_DEPTH + ")"); } String rawEncodingId = encodingSpecs.get(fbs.encoding()); + if (rawEncodingId.isBlank()) { + // EncodingId.parse rejects blank ids with IllegalArgumentException; the file is + // untrusted input, so a blank spec entry must surface as VortexException instead. + throw new VortexException("blank encoding id at array spec index " + fbs.encoding()); + } ArrayNode[] children = new ArrayNode[fbs.childrenLength()]; for (int i = 0; i < children.length; i++) { @@ -99,11 +102,6 @@ private static ArrayNode convertArrayNode( } MemorySegment meta = fbs.metadataAsSegment(); - return switch (EncodingId.parse(rawEncodingId)) { - case EncodingId.WellKnown wellKnown -> - new KnownArrayNode(wellKnown, meta, children, bufferIndices); - case EncodingId.Custom custom -> - new UnknownArrayNode(custom.id(), meta, children, bufferIndices); - }; + return new ArrayNode(EncodingId.parse(rawEncodingId), meta, children, bufferIndices); } } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ReadRegistry.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ReadRegistry.java index 09070597..0274172d 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ReadRegistry.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ReadRegistry.java @@ -7,8 +7,6 @@ import io.github.dfa1.vortex.reader.decode.ArrayNode; import io.github.dfa1.vortex.reader.decode.DecodeContext; import io.github.dfa1.vortex.reader.decode.EncodingDecoder; -import io.github.dfa1.vortex.reader.decode.KnownArrayNode; -import io.github.dfa1.vortex.reader.decode.UnknownArrayNode; import java.lang.foreign.MemorySegment; import java.util.Collections; @@ -76,21 +74,14 @@ public boolean hasDecoder(EncodingId encodingId) { /// @return the decoded [Array] public Array decode(DecodeContext ctx) { ArrayNode node = ctx.node(); - EncodingDecoder decoder = switch (node) { - case KnownArrayNode k -> decoders.get(k.encodingId().id()); - case UnknownArrayNode _ -> null; - }; + EncodingDecoder decoder = decoders.get(node.encodingId().id()); if (decoder != null) { return decoder.decode(ctx); } if (allowUnknown) { return decodeUnknown(ctx, node); } - String id = switch (node) { - case KnownArrayNode k -> k.encodingId().id(); - case UnknownArrayNode u -> u.rawEncodingId(); - }; - throw new VortexException("no decoder registered for " + id); + throw new VortexException("no decoder registered for " + node.encodingId().id()); } /// Decodes the array described by `ctx` and returns its primary backing segment. @@ -99,25 +90,16 @@ public Array decode(DecodeContext ctx) { /// @return the primary [MemorySegment] of the decoded array public MemorySegment decodeAsSegment(DecodeContext ctx) { ArrayNode node = ctx.node(); - EncodingDecoder decoder = switch (node) { - case KnownArrayNode k -> decoders.get(k.encodingId().id()); - case UnknownArrayNode _ -> null; - }; + EncodingDecoder decoder = decoders.get(node.encodingId().id()); if (decoder != null) { return decoder.decode(ctx).materialize(ctx.arena()); } - String id = switch (node) { - case KnownArrayNode k -> k.encodingId().id(); - case UnknownArrayNode u -> u.rawEncodingId(); - }; - throw new VortexException("no decoder registered for " + id + " (or encoding has no primary segment)"); + throw new VortexException("no decoder registered for " + node.encodingId().id() + + " (or encoding has no primary segment)"); } private static UnknownArray decodeUnknown(DecodeContext ctx, ArrayNode node) { - String rawId = switch (node) { - case KnownArrayNode k -> k.encodingId().id(); - case UnknownArrayNode u -> u.rawEncodingId(); - }; + String rawId = node.encodingId().id(); MemorySegment[] bufs = new MemorySegment[node.bufferIndices().length]; for (int i = 0; i < bufs.length; i++) { bufs[i] = ctx.buffer(i); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ArrayNode.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ArrayNode.java index 520d6ab1..26ed8876 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ArrayNode.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ArrayNode.java @@ -4,40 +4,26 @@ import java.lang.foreign.MemorySegment; -/// Encoded array node as stored in a Flat layout segment. -/// In-file representation before decoding; mirrors the Go ArrayNode struct. +/// Encoded array node as stored in a Flat layout segment — the in-file representation before +/// decoding. /// -/// Sealed: a node is either [KnownArrayNode] (id resolves to an [EncodingId.WellKnown]) or -/// [UnknownArrayNode] (id is an arbitrary string only meaningful for -/// [io.github.dfa1.vortex.reader.ReadRegistry#isAllowUnknown()] passthrough decode). -public sealed interface ArrayNode permits KnownArrayNode, UnknownArrayNode { - - /// Factory that builds the right node kind for `encodingId`: a [KnownArrayNode] for a - /// well-known id, or an [UnknownArrayNode] for a [EncodingId.Custom] one. - /// Mostly used by tests and helper code that converts an `EncodeNode` tree back into - /// an `ArrayNode` tree. - /// - /// @param encodingId the encoding identifier - /// @param metadata encoding-specific metadata bytes, or `null` - /// @param children child nodes - /// @param bufferIndices segment buffer indices for this node - /// @return a [KnownArrayNode] for a well-known id, else an [UnknownArrayNode] - static ArrayNode of(EncodingId encodingId, MemorySegment metadata, ArrayNode[] children, - int[] bufferIndices) { - return switch (encodingId) { - case EncodingId.WellKnown wellKnown -> - new KnownArrayNode(wellKnown, metadata, children, bufferIndices); - case EncodingId.Custom custom -> - new UnknownArrayNode(custom.id(), metadata, children, bufferIndices); - }; - } - - /// @return encoding-specific metadata bytes, or `null` - MemorySegment metadata(); - - /// @return child nodes - ArrayNode[] children(); - - /// @return segment buffer indices for this node - int[] bufferIndices(); +/// The encoding id is carried typed, as either an [EncodingId.WellKnown] constant or an +/// [EncodingId.Custom] wrapping a raw wire string, so the Known/Unknown distinction lives in the id +/// itself rather than in a node subtype. Whether an id is decodable is a separate, +/// decode-time question owned by the [io.github.dfa1.vortex.reader.ReadRegistry], not a property of +/// the node: an [EncodingId.Custom] id, or a [EncodingId.WellKnown] id no registered decoder +/// claims, either decodes as a passthrough [io.github.dfa1.vortex.reader.array.UnknownArray] (when +/// [io.github.dfa1.vortex.reader.ReadRegistry#isAllowUnknown()] is set) or fails the decode. +/// +/// @param encodingId the encoding identifier, [EncodingId.WellKnown] or [EncodingId.Custom] +/// @param metadata encoding-specific metadata bytes, or `null` +/// @param children child nodes +/// @param bufferIndices segment buffer indices for this node +@SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. +public record ArrayNode( + EncodingId encodingId, + MemorySegment metadata, + ArrayNode[] children, + int[] bufferIndices +) { } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/KnownArrayNode.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/KnownArrayNode.java deleted file mode 100644 index f0aadcef..00000000 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/KnownArrayNode.java +++ /dev/null @@ -1,20 +0,0 @@ -package io.github.dfa1.vortex.reader.decode; - -import io.github.dfa1.vortex.core.model.EncodingId; - -import java.lang.foreign.MemorySegment; - -/// Array node whose encoding id is well-known to this build (an [EncodingId.WellKnown] constant). -/// -/// @param encodingId well-known encoding id -/// @param metadata encoding-specific metadata bytes, or `null` -/// @param children child nodes -/// @param bufferIndices segment buffer indices -@SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. -public record KnownArrayNode( - EncodingId.WellKnown encodingId, - MemorySegment metadata, - ArrayNode[] children, - int[] bufferIndices -) implements ArrayNode { -} diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/UnknownArrayNode.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/UnknownArrayNode.java deleted file mode 100644 index 2804d04c..00000000 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/UnknownArrayNode.java +++ /dev/null @@ -1,21 +0,0 @@ -package io.github.dfa1.vortex.reader.decode; - -import java.lang.foreign.MemorySegment; - -/// Array node whose encoding id is not a recognized [io.github.dfa1.vortex.core.model.EncodingId]. -/// Produced when a file uses an encoding this build does not know about. Decoded as -/// [io.github.dfa1.vortex.reader.array.UnknownArray] when -/// [io.github.dfa1.vortex.reader.ReadRegistry#isAllowUnknown()] is set; otherwise the decode call throws. -/// -/// @param rawEncodingId the raw encoding id string from the file -/// @param metadata encoding-specific metadata bytes, or `null` -/// @param children child nodes -/// @param bufferIndices segment buffer indices -@SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. -public record UnknownArrayNode( - String rawEncodingId, - MemorySegment metadata, - ArrayNode[] children, - int[] bufferIndices -) implements ArrayNode { -} diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/FlatSegmentDecoderDecodeTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/FlatSegmentDecoderDecodeTest.java index 1aa40de1..1014a6dd 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/FlatSegmentDecoderDecodeTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/FlatSegmentDecoderDecodeTest.java @@ -3,6 +3,7 @@ import io.github.dfa1.vortex.core.fbs.FbsArrayNode; import io.github.dfa1.vortex.core.fbs.FbsBuffer; import io.github.dfa1.vortex.core.fbs.FbsBuilder; +import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.UnknownArray; @@ -14,15 +15,16 @@ import static io.github.dfa1.vortex.core.io.PTypeIO.LE_INT; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowable; /// Successful flat-segment decode path — complements [FlatSegmentBoundsSecurityTest] (which only /// drives the rejection paths). A buffer descriptor with non-zero padding exercises the offset -/// walk, and an unknown encoding id exercises the `UnknownArrayNode` fallback through an +/// walk, and an unknown encoding id exercises the unknown-id passthrough through an /// allow-unknown registry. Together these pin two otherwise-untested spots: /// - the `dataOffset += padding` accumulation: with padding > 0, flipping `+=` to `-=` slices at a /// negative offset and fails, so a clean decode proves the addition. -/// - the `orElseGet(() -> new UnknownArrayNode(...))` fallback: returning `null` there yields a -/// null node and the decode would not produce an `UnknownArray`. +/// - the unknown-id node construction: mishandling an unresolvable id there yields a +/// node the decode would not turn into an `UnknownArray`. class FlatSegmentDecoderDecodeTest { @Test @@ -46,11 +48,36 @@ void decode_unknownEncodingWithBufferPadding_returnsUnknownArray() { DType.I32, 0, arena); // Then — the allow-unknown path produced an UnknownArray (proves both the +padding - // walk and the UnknownArrayNode fallback ran) + // walk and the unknown-id passthrough ran) assertThat(result).isInstanceOf(UnknownArray.class); } } + @Test + void decode_blankEncodingId_throwsVortexException() { + ReadRegistry registry = ReadRegistry.builder().allowUnknown().build(); + FlatSegmentDecoder sut = new FlatSegmentDecoder(registry); + + try (Arena arena = Arena.ofConfined()) { + // Given — a zero-length FlatBuffer string in the spec table decodes to "", which + // EncodingId.parse rejects with IllegalArgumentException; untrusted input must + // surface as VortexException instead, even under allowUnknown + byte[] fb = arrayFlatBufferOneBuffer(0, 0L); + MemorySegment seg = arena.allocate((long) fb.length + 4); + MemorySegment.copy(MemorySegment.ofArray(fb), 0, seg, 0, fb.length); + seg.set(LE_INT, fb.length, fb.length); + + // When + Throwable result = catchThrowable(() -> sut.decode(seg, List.of(""), + DType.I32, 0, arena)); + + // Then + assertThat(result) + .isInstanceOf(VortexException.class) + .hasMessageContaining("blank encoding id"); + } + } + /// Builds an `Array` FlatBuffer with a single buffer descriptor of the given padding/length. private static byte[] arrayFlatBufferOneBuffer(int padding, long length) { FbsBuilder b = new FbsBuilder(); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ReadRegistryTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ReadRegistryTest.java index f82b91cd..1a295f4c 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/ReadRegistryTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ReadRegistryTest.java @@ -8,7 +8,6 @@ import io.github.dfa1.vortex.reader.decode.ArrayNode; import io.github.dfa1.vortex.reader.decode.DecodeContext; import io.github.dfa1.vortex.reader.decode.EncodingDecoder; -import io.github.dfa1.vortex.reader.decode.UnknownArrayNode; import org.junit.jupiter.api.Test; import java.lang.foreign.Arena; @@ -25,7 +24,7 @@ class ReadRegistryTest { void decodeUnknownEncodingThrowsByDefault() { // Given ReadRegistry sut = ReadRegistry.empty(); - ArrayNode node = new UnknownArrayNode("some.unknown", + ArrayNode node = new ArrayNode(EncodingId.parse("some.unknown"), MemorySegment.ofArray(new byte[0]), new ArrayNode[0], new int[0]); DecodeContext ctx = new DecodeContext(node, DTypes.I32, 0L, new MemorySegment[0], sut, Arena.ofAuto()); @@ -40,7 +39,7 @@ void decodeUnknownEncodingThrowsByDefault() { void decodeKnownEncodingWithoutDecoderThrowsByDefault() { // Given — EncodingId is known but no decoder registered for it ReadRegistry sut = ReadRegistry.empty(); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, + ArrayNode node = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, MemorySegment.ofArray(new byte[0]), new ArrayNode[0], new int[0]); DecodeContext ctx = new DecodeContext(node, DTypes.I32, 0L, new MemorySegment[0], sut, Arena.ofAuto()); @@ -55,7 +54,7 @@ void decodeKnownEncodingWithoutDecoderThrowsByDefault() { void decodeKnownEncodingWithoutDecoderReturnsUnknownArrayWhenAllowed() { // Given ReadRegistry sut = ReadRegistry.builder().allowUnknown().build(); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, + ArrayNode node = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, MemorySegment.ofArray(new byte[0]), new ArrayNode[0], new int[0]); DecodeContext ctx = new DecodeContext(node, DTypes.I32, 0L, new MemorySegment[0], sut, Arena.ofAuto()); @@ -75,7 +74,7 @@ void decodeUnknownEncodingReturnsUnknownArrayWhenAllowed() { MemorySegment metadata = MemorySegment.ofArray(new byte[]{1, 2, 3}); MemorySegment buf = Arena.ofAuto().allocate(4); buf.set(java.lang.foreign.ValueLayout.JAVA_INT, 0, 42); - ArrayNode node = new UnknownArrayNode("some.unknown", + ArrayNode node = new ArrayNode(EncodingId.parse("some.unknown"), metadata, new ArrayNode[0], new int[]{0}); DecodeContext ctx = new DecodeContext(node, DTypes.I32, 5L, new MemorySegment[]{buf}, sut, Arena.ofAuto()); @@ -101,9 +100,9 @@ void decodeUnknownEncodingWrapsChildrenAsUnknown() { ReadRegistry sut = ReadRegistry.builder().allowUnknown().build(); // Child uses a known id; allow-unknown still wraps it unknown because // its parent is unknown — mirrors Rust decode_foreign in vortex-array/src/serde.rs:380. - ArrayNode child = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, + ArrayNode child = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, MemorySegment.ofArray(new byte[0]), new ArrayNode[0], new int[0]); - ArrayNode parent = new UnknownArrayNode("some.unknown", + ArrayNode parent = new ArrayNode(EncodingId.parse("some.unknown"), MemorySegment.ofArray(new byte[0]), new ArrayNode[]{child}, new int[0]); DecodeContext ctx = new DecodeContext(parent, DTypes.I32, 0L, new MemorySegment[0], sut, Arena.ofAuto()); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java index 07d310d5..7419d15a 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoderTest.java @@ -56,7 +56,7 @@ private static MemorySegment leDoubles(double... vs) { @Test void decode_nonPrimitiveDtype_throws() { // Given a Utf8 dtype on an ALP node - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ALP, MemorySegment.ofArray(new ProtoALPMetadata(0, 0, null).encode()), + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ALP, MemorySegment.ofArray(new ProtoALPMetadata(0, 0, null).encode()), new ArrayNode[0], new int[0]); DecodeContext ctx = new DecodeContext(node, DType.UTF8, 1, new MemorySegment[0], REGISTRY, Arena.ofAuto()); @@ -68,8 +68,8 @@ void decode_nonPrimitiveDtype_throws() { @Test void decode_missingMetadata_defaultsToZeroExponents() { // Given no metadata — decoder falls back to exp_e=0, exp_f=0 (scale 1.0) - ArrayNode enc = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ALP, null, new ArrayNode[]{enc}, new int[0]); + ArrayNode enc = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ALP, null, new ArrayNode[]{enc}, new int[0]); DecodeContext ctx = new DecodeContext(node, F64, 2, new MemorySegment[]{leLongs(5L, 7L)}, REGISTRY, Arena.ofAuto()); // When @@ -84,9 +84,9 @@ void decode_missingMetadata_defaultsToZeroExponents() { void decode_f64_broadcastNoPatches_returnsConstant() { // Given a single encoded value but 4 logical rows (capacity < n) and no patches: // the decoder broadcasts it into a constant array - ArrayNode enc = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); + ArrayNode enc = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); byte[] meta = new ProtoALPMetadata(2, 0, null).encode(); // exp_e=2 -> *0.01 - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ALP, MemorySegment.ofArray(meta), new ArrayNode[]{enc}, new int[0]); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ALP, MemorySegment.ofArray(meta), new ArrayNode[]{enc}, new int[0]); DecodeContext ctx = new DecodeContext(node, F64, 4, new MemorySegment[]{leLongs(123L)}, REGISTRY, Arena.ofAuto()); // When @@ -102,9 +102,9 @@ void decode_f64_broadcastNoPatches_returnsConstant() { @Test void decode_f32_broadcastNoPatches_returnsConstant() { // Given single value, 3 rows, no patches - ArrayNode enc = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); + ArrayNode enc = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); byte[] meta = new ProtoALPMetadata(1, 0, null).encode(); // exp_e=1 -> *0.1 - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ALP, MemorySegment.ofArray(meta), new ArrayNode[]{enc}, new int[0]); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ALP, MemorySegment.ofArray(meta), new ArrayNode[]{enc}, new int[0]); DecodeContext ctx = new DecodeContext(node, F32, 3, new MemorySegment[]{leInts(25)}, REGISTRY, Arena.ofAuto()); // When @@ -124,10 +124,10 @@ void decode_f64_patches_withU8Indices() { ProtoPatchesMetadata pm = new ProtoPatchesMetadata(1L, 0L, io.github.dfa1.vortex.core.proto.ProtoPType.U8, null, null, null); byte[] meta = new ProtoALPMetadata(2, 0, pm).encode(); // *0.01 - ArrayNode enc = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); - ArrayNode idx = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); - ArrayNode val = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{2}); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ALP, MemorySegment.ofArray(meta), + ArrayNode enc = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); + ArrayNode idx = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); + ArrayNode val = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{2}); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ALP, MemorySegment.ofArray(meta), new ArrayNode[]{enc, idx, val}, new int[0]); MemorySegment idxSeg = MemorySegment.ofArray(new byte[]{1}); // patch row 1 @@ -149,10 +149,10 @@ void decode_patches_nonUnsignedIndexPtype_throws() { ProtoPatchesMetadata pm = new ProtoPatchesMetadata(1L, 0L, io.github.dfa1.vortex.core.proto.ProtoPType.I32, null, null, null); byte[] meta = new ProtoALPMetadata(2, 0, pm).encode(); - ArrayNode enc = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); - ArrayNode idx = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); - ArrayNode val = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{2}); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ALP, MemorySegment.ofArray(meta), + ArrayNode enc = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); + ArrayNode idx = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); + ArrayNode val = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{2}); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ALP, MemorySegment.ofArray(meta), new ArrayNode[]{enc, idx, val}, new int[0]); MemorySegment[] segs = {leLongs(100L, 0L), leInts(1), leDoubles(9.0)}; diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ByteBoolEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ByteBoolEncodingDecoderTest.java index 0e5f86dd..184b6578 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ByteBoolEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ByteBoolEncodingDecoderTest.java @@ -29,7 +29,7 @@ static Stream cases() { private static DecodeContext buildCtx(byte[] byteValues) { MemorySegment buf = MemorySegment.ofArray(byteValues); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_BYTEBOOL, null, new ArrayNode[0], new int[]{0}); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_BYTEBOOL, null, new ArrayNode[0], new int[]{0}); ReadRegistry registry = TestRegistry.ofDecoders(new ByteBoolEncodingDecoder()); return new DecodeContext(node, DTypes.BOOL, byteValues.length, new MemorySegment[]{buf}, registry, Arena.ofAuto()); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DateTimePartsEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DateTimePartsEncodingDecoderTest.java index 43bbff41..4f62a654 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DateTimePartsEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DateTimePartsEncodingDecoderTest.java @@ -39,10 +39,10 @@ private static DType timestampDType(TimeUnit unit, boolean nullable) { /// Builds a context with three I64 part-children backed by the given segments. private static DecodeContext ctx(MemorySegment meta, DType dtype, long n, MemorySegment days, MemorySegment seconds, MemorySegment subseconds) { - ArrayNode d = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); - ArrayNode s = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); - ArrayNode ss = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{2}); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DATETIMEPARTS, meta, new ArrayNode[]{d, s, ss}, new int[0]); + ArrayNode d = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); + ArrayNode s = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); + ArrayNode ss = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{2}); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_DATETIMEPARTS, meta, new ArrayNode[]{d, s, ss}, new int[0]); return new DecodeContext(node, dtype, n, new MemorySegment[]{days, seconds, subseconds}, REGISTRY, Arena.ofAuto()); } @@ -55,7 +55,7 @@ void encodingId_isVortexDateTimeParts() { @Test void decode_missingMetadata_throws() { // Given a node with no metadata - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DATETIMEPARTS, null, new ArrayNode[0], new int[0]); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_DATETIMEPARTS, null, new ArrayNode[0], new int[0]); DecodeContext c = new DecodeContext(node, timestampDType(TimeUnit.Milliseconds, false), 1, new MemorySegment[0], REGISTRY, Arena.ofAuto()); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DecimalEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DecimalEncodingDecoderTest.java index 6e55fe4f..eb38a965 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DecimalEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DecimalEncodingDecoderTest.java @@ -31,7 +31,7 @@ private static Array decode(int valuesType, int rowCount, int bufferBytes) { private static Array decode(MemorySegment meta, int rowCount, int bufferBytes) { MemorySegment buf = MemorySegment.ofArray(new byte[bufferBytes]); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DECIMAL, meta, new ArrayNode[0], new int[]{0}); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_DECIMAL, meta, new ArrayNode[0], new int[]{0}); DecodeContext ctx = new DecodeContext(node, DECIMAL, rowCount, new MemorySegment[]{buf}, ReadRegistry.empty(), Arena.ofAuto()); return SUT.decode(ctx); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoderTest.java index 59616f33..973d69fb 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoderTest.java @@ -37,7 +37,7 @@ void decode_nullMetadata_returnsEmptyArray(PType ptype) { // Given no metadata — the decoder defaults to deltas_len=0 and short-circuits // to an empty array of the right ptype (a path the encoder never emits, since it // always writes metadata) - ArrayNode node = ArrayNode.of(EncodingId.FASTLANES_DELTA, null, new ArrayNode[0], new int[0]); + ArrayNode node = new ArrayNode(EncodingId.FASTLANES_DELTA, null, new ArrayNode[0], new int[0]); DecodeContext ctx = new DecodeContext(node, new DType.Primitive(ptype, false), 0, new MemorySegment[0], REGISTRY, Arena.ofAuto()); @@ -58,9 +58,9 @@ void decode_constantChildren_broadcastsAcrossChunk() { long deltasLen = FL_CHUNK_SIZE; MemorySegment meta = MemorySegment.ofArray(new ProtoDeltaMetadata(deltasLen, 0).encode()); - ArrayNode bases = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); - ArrayNode deltas = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); - ArrayNode node = ArrayNode.of(EncodingId.FASTLANES_DELTA, meta, new ArrayNode[]{bases, deltas}, new int[0]); + ArrayNode bases = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); + ArrayNode deltas = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); + ArrayNode node = new ArrayNode(EncodingId.FASTLANES_DELTA, meta, new ArrayNode[]{bases, deltas}, new int[0]); // one element each → broadcast MemorySegment[] segs = {TestSegments.leLongs(0L), TestSegments.leLongs(0L)}; @@ -83,9 +83,9 @@ void decode_constantBases_nonZeroOffsetAndBase() { PType ptype = PType.I64; MemorySegment meta = MemorySegment.ofArray(new ProtoDeltaMetadata(FL_CHUNK_SIZE, 0).encode()); - ArrayNode bases = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); - ArrayNode deltas = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); - ArrayNode node = ArrayNode.of(EncodingId.FASTLANES_DELTA, meta, new ArrayNode[]{bases, deltas}, new int[0]); + ArrayNode bases = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); + ArrayNode deltas = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); + ArrayNode node = new ArrayNode(EncodingId.FASTLANES_DELTA, meta, new ArrayNode[]{bases, deltas}, new int[0]); MemorySegment[] segs = {TestSegments.leLongs(5L), TestSegments.leLongs(0L)}; DecodeContext ctx = new DecodeContext(node, new DType.Primitive(ptype, false), 3, segs, REGISTRY, Arena.ofAuto()); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java index bc9ee3e4..bdab8a31 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoderTest.java @@ -136,7 +136,7 @@ void unsupportedValuePType_throws() { @Test void missingMetadata_throws() { // Given — primitive dict with no metadata - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DICT, null, new ArrayNode[0], new int[]{}); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_DICT, null, new ArrayNode[0], new int[]{}); DecodeContext ctx = new DecodeContext(node, DType.I32, 1, new MemorySegment[0], REGISTRY, Arena.ofAuto()); @@ -149,7 +149,7 @@ void missingMetadata_throws() { @Test void emptyMetadata_throws() { // Given — metadata present but with zero remaining bytes (exercises !hasRemaining) - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DICT, MemorySegment.ofArray(new byte[0]), + ArrayNode node = new ArrayNode(EncodingId.VORTEX_DICT, MemorySegment.ofArray(new byte[0]), new ArrayNode[0], new int[]{}); DecodeContext ctx = new DecodeContext(node, DType.I32, 1, new MemorySegment[0], REGISTRY, Arena.ofAuto()); @@ -164,7 +164,7 @@ void emptyMetadata_throws() { void malformedProtoMetadata_throws() { // Given — >1 byte (routes to proto path) but a truncated varint that proto decode rejects MemorySegment meta = MemorySegment.ofArray(new byte[]{0x08, (byte) 0x80}); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DICT, meta, + ArrayNode node = new ArrayNode(EncodingId.VORTEX_DICT, meta, new ArrayNode[]{primitiveNode(0), primitiveNode(1)}, new int[]{}); DecodeContext ctx = new DecodeContext(node, DType.I32, 1, new MemorySegment[]{u8Codes(0), TestSegments.leLongs(0)}, REGISTRY, Arena.ofAuto()); @@ -298,7 +298,7 @@ void legacyLayout_decodesStringsByCode() { MemorySegment codes = u8Codes(1, 0, 1); MemorySegment meta = MemorySegment.ofArray(new byte[]{(byte) PType.U8.ordinal()}); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DICT, meta, new ArrayNode[0], new int[]{0, 1, 2}); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_DICT, meta, new ArrayNode[0], new int[]{0, 1, 2}); DecodeContext ctx = new DecodeContext(node, DType.UTF8, 3, new MemorySegment[]{bytes, offsets, codes}, REGISTRY, Arena.ofAuto()); @@ -326,9 +326,9 @@ void protoLayout_decodesStringsByCode() { ArrayNode codesNode = primitiveNode(0); ArrayNode offsetsNode = primitiveNode(2); - ArrayNode valuesNode = ArrayNode.of(EncodingId.VORTEX_VARBIN, varBinMeta, + ArrayNode valuesNode = new ArrayNode(EncodingId.VORTEX_VARBIN, varBinMeta, new ArrayNode[]{offsetsNode}, new int[]{1}); - ArrayNode dictNode = ArrayNode.of(EncodingId.VORTEX_DICT, dictMeta, + ArrayNode dictNode = new ArrayNode(EncodingId.VORTEX_DICT, dictMeta, new ArrayNode[]{codesNode, valuesNode}, new int[]{}); DecodeContext ctx = new DecodeContext(dictNode, DType.UTF8, 3, @@ -346,7 +346,7 @@ void protoLayout_decodesStringsByCode() { @Test void legacyLayout_missingMetadata_throws() { // Given — no children and no metadata - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DICT, null, new ArrayNode[0], new int[]{}); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_DICT, null, new ArrayNode[0], new int[]{}); DecodeContext ctx = new DecodeContext(node, DType.UTF8, 0, new MemorySegment[0], REGISTRY, Arena.ofAuto()); @@ -361,7 +361,7 @@ void protoLayout_malformedMetadata_throws() { // Given — children present, metadata is an invalid (truncated varint) proto blob MemorySegment meta = MemorySegment.ofArray(new byte[]{0x08, (byte) 0x80}); ArrayNode child = primitiveNode(0); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DICT, meta, + ArrayNode node = new ArrayNode(EncodingId.VORTEX_DICT, meta, new ArrayNode[]{child, child}, new int[]{}); DecodeContext ctx = new DecodeContext(node, DType.UTF8, 1, new MemorySegment[]{u8Codes(0)}, REGISTRY, Arena.ofAuto()); @@ -376,7 +376,7 @@ void protoLayout_malformedMetadata_throws() { void protoLayout_missingMetadata_throws() { // Given — children present but metadata absent ArrayNode child = primitiveNode(0); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DICT, null, new ArrayNode[]{child, child}, new int[]{}); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_DICT, null, new ArrayNode[]{child, child}, new int[]{}); DecodeContext ctx = new DecodeContext(node, DType.UTF8, 1, new MemorySegment[]{u8Codes(0)}, REGISTRY, Arena.ofAuto()); @@ -389,7 +389,7 @@ void protoLayout_missingMetadata_throws() { @Test void legacyLayout_emptyMetadata_throws() { // Given — no children and zero-remaining metadata (exercises !hasRemaining) - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DICT, MemorySegment.ofArray(new byte[0]), + ArrayNode node = new ArrayNode(EncodingId.VORTEX_DICT, MemorySegment.ofArray(new byte[0]), new ArrayNode[0], new int[]{}); DecodeContext ctx = new DecodeContext(node, DType.UTF8, 0, new MemorySegment[0], REGISTRY, Arena.ofAuto()); @@ -404,7 +404,7 @@ void legacyLayout_emptyMetadata_throws() { void protoLayout_emptyMetadata_throws() { // Given — children present, zero-remaining metadata ArrayNode child = primitiveNode(0); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_DICT, MemorySegment.ofArray(new byte[0]), + ArrayNode node = new ArrayNode(EncodingId.VORTEX_DICT, MemorySegment.ofArray(new byte[0]), new ArrayNode[]{child, child}, new int[]{}); DecodeContext ctx = new DecodeContext(node, DType.UTF8, 1, new MemorySegment[]{u8Codes(0)}, REGISTRY, Arena.ofAuto()); @@ -442,7 +442,7 @@ private static Array decodeProtoSegments(DType dtype, PType codePType, MemorySeg MemorySegment values, int valuesLen, int rowCount) { MemorySegment meta = MemorySegment.ofArray(new ProtoDictMetadata(valuesLen, protoPType(codePType), null, null).encode()); MemorySegment[] segs = {codes, values}; - ArrayNode dictNode = ArrayNode.of(EncodingId.VORTEX_DICT, meta, + ArrayNode dictNode = new ArrayNode(EncodingId.VORTEX_DICT, meta, new ArrayNode[]{primitiveNode(0), primitiveNode(1)}, new int[]{}); DecodeContext ctx = new DecodeContext(dictNode, dtype, rowCount, segs, REGISTRY, Arena.ofAuto()); return SUT.decode(ctx); @@ -452,14 +452,14 @@ private static Array decodeLegacy(DType dtype, PType codePType, MemorySegment va MemorySegment codes, int rowCount) { MemorySegment meta = MemorySegment.ofArray(new byte[]{(byte) codePType.ordinal()}); MemorySegment[] segs = {values, codes}; - ArrayNode dictNode = ArrayNode.of(EncodingId.VORTEX_DICT, meta, + ArrayNode dictNode = new ArrayNode(EncodingId.VORTEX_DICT, meta, new ArrayNode[]{primitiveNode(0), primitiveNode(1)}, new int[]{}); DecodeContext ctx = new DecodeContext(dictNode, dtype, rowCount, segs, REGISTRY, Arena.ofAuto()); return SUT.decode(ctx); } private static ArrayNode primitiveNode(int bufferIndex) { - return ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{bufferIndex}); + return new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{bufferIndex}); } private static io.github.dfa1.vortex.core.proto.ProtoPType protoPType(PType core) { diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/NullEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/NullEncodingDecoderTest.java index ade733ec..0820bcf5 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/NullEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/NullEncodingDecoderTest.java @@ -18,7 +18,7 @@ class NullEncodingDecoderTest { void decode_nullArray_returnsNullArrayWithCorrectLength() { // Given long rowCount = 42L; - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_NULL, null, new ArrayNode[0], new int[0]); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_NULL, null, new ArrayNode[0], new int[0]); DecodeContext ctx = new DecodeContext(node, DTypes.NULL, rowCount, new MemorySegment[0], ReadRegistry.empty(), Arena.ofAuto()); var sut = new NullEncodingDecoder(); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoderTest.java index bd3d4b7c..f9e588ca 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoderTest.java @@ -48,12 +48,12 @@ private static Array decode(DType dtype, int n, MemorySegment[] segments = {inner, laneOffsets, patchIndices, patchValues}; - ArrayNode innerNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); - ArrayNode laneNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); - ArrayNode idxNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{2}); - ArrayNode valNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{3}); + ArrayNode innerNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); + ArrayNode laneNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); + ArrayNode idxNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{2}); + ArrayNode valNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{3}); - ArrayNode patchedNode = ArrayNode.of(EncodingId.VORTEX_PATCHED, meta, + ArrayNode patchedNode = new ArrayNode(EncodingId.VORTEX_PATCHED, meta, new ArrayNode[]{innerNode, laneNode, idxNode, valNode}, new int[]{}); DecodeContext ctx = new DecodeContext(patchedNode, dtype, n, segments, REGISTRY, Arena.ofAuto()); @@ -139,8 +139,8 @@ void decode_i64_singlePatch() { @Test void decode_missingMetadata_throws() { // Given - ArrayNode innerNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); - ArrayNode patchedNode = ArrayNode.of(EncodingId.VORTEX_PATCHED, null, + ArrayNode innerNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); + ArrayNode patchedNode = new ArrayNode(EncodingId.VORTEX_PATCHED, null, new ArrayNode[]{innerNode, innerNode, innerNode, innerNode}, new int[]{}); MemorySegment seg = TestSegments.leInts(1, 2, 3); DecodeContext ctx = new DecodeContext(patchedNode, DType.I32, 3, diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PcoEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PcoEncodingDecoderTest.java index fe79f709..a166bad4 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PcoEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PcoEncodingDecoderTest.java @@ -39,7 +39,7 @@ private static MemorySegment validMetaBuffer() { } private static DecodeContext ctxWith(MemorySegment meta, DType dtype, long rowCount, MemorySegment[] buffers) { - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_PCO, meta, new ArrayNode[0], + ArrayNode node = new ArrayNode(EncodingId.VORTEX_PCO, meta, new ArrayNode[0], bufferIndices(buffers.length)); return new DecodeContext(node, dtype, rowCount, buffers, ReadRegistry.empty(), Arena.ofAuto()); } @@ -50,14 +50,14 @@ private static DecodeContext ctxWithValidity(MemorySegment meta, DType dtype, lo allBuffers[0] = validityBuf; System.arraycopy(pcoBuffers, 0, allBuffers, 1, pcoBuffers.length); - ArrayNode validityNode = ArrayNode.of(EncodingId.VORTEX_BOOL, null, new ArrayNode[0], + ArrayNode validityNode = new ArrayNode(EncodingId.VORTEX_BOOL, null, new ArrayNode[0], new int[]{0}); int[] pcoBufferIndices = new int[pcoBuffers.length]; for (int i = 0; i < pcoBuffers.length; i++) { pcoBufferIndices[i] = i + 1; } - ArrayNode pcoNode = ArrayNode.of(EncodingId.VORTEX_PCO, meta, new ArrayNode[]{validityNode}, + ArrayNode pcoNode = new ArrayNode(EncodingId.VORTEX_PCO, meta, new ArrayNode[]{validityNode}, pcoBufferIndices); ReadRegistry registry = TestRegistry.ofDecoders(new BoolEncodingDecoder()); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoderTest.java index edd6e99e..dac19da2 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VarBinEncodingDecoderTest.java @@ -26,8 +26,8 @@ private static MemorySegment i32OffsetsMeta() { private static DecodeContext ctx(MemorySegment meta, MemorySegment bytes, MemorySegment offsets, long n) { // children[0] = offsets (primitive, segment index 1); bufferIndices[0] -> bytes (index 0) - ArrayNode offsetsNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); - ArrayNode varbinNode = ArrayNode.of(EncodingId.VORTEX_VARBIN, meta, new ArrayNode[]{offsetsNode}, new int[]{0}); + ArrayNode offsetsNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); + ArrayNode varbinNode = new ArrayNode(EncodingId.VORTEX_VARBIN, meta, new ArrayNode[]{offsetsNode}, new int[]{0}); return new DecodeContext(varbinNode, DType.UTF8, n, new MemorySegment[]{bytes, offsets}, REGISTRY, Arena.ofAuto()); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VarBinViewEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VarBinViewEncodingDecoderTest.java index a944b680..8efc8dd9 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VarBinViewEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VarBinViewEncodingDecoderTest.java @@ -36,7 +36,7 @@ void decode_binaryDtype_inlineViews() { writeInlineView(views, 0, a); writeInlineView(views, 1, b); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_VARBINVIEW, null, new ArrayNode[0], new int[]{0}); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_VARBINVIEW, null, new ArrayNode[0], new int[]{0}); DecodeContext ctx = new DecodeContext(node, DType.BINARY, 2, new MemorySegment[]{views}, ReadRegistry.empty(), arena); @@ -53,7 +53,7 @@ void decode_binaryDtype_inlineViews() { @Test void decode_wrongDtype_throws() { // Given a primitive dtype - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_VARBINVIEW, null, new ArrayNode[0], new int[]{0}); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_VARBINVIEW, null, new ArrayNode[0], new int[]{0}); DecodeContext ctx = new DecodeContext(node, new DType.Primitive(io.github.dfa1.vortex.core.model.PType.I32, false), 0, new MemorySegment[]{Arena.ofAuto().allocate(16)}, ReadRegistry.empty(), Arena.ofAuto()); @@ -65,7 +65,7 @@ void decode_wrongDtype_throws() { @Test void decode_noBuffers_throws() { // Given a node with zero buffer indices - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_VARBINVIEW, null, new ArrayNode[0], new int[0]); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_VARBINVIEW, null, new ArrayNode[0], new int[0]); DecodeContext ctx = new DecodeContext(node, DType.UTF8, 0, new MemorySegment[0], ReadRegistry.empty(), Arena.ofAuto()); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoderTest.java index 8a3b0ffa..5a194d81 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VariantEncodingDecoderTest.java @@ -33,11 +33,11 @@ private static MemorySegment variantMetaWithShredded(io.github.dfa1.vortex.core. } private static ArrayNode primitiveChildNode(int segIdx) { - return ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{segIdx}); + return new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{segIdx}); } private static ArrayNode nullChildNode() { - return ArrayNode.of(EncodingId.VORTEX_NULL, null, new ArrayNode[0], new int[]{}); + return new ArrayNode(EncodingId.VORTEX_NULL, null, new ArrayNode[0], new int[]{}); } @@ -45,7 +45,7 @@ private static ArrayNode nullChildNode() { void decode_withoutShredded_returnsCoreStorageOnly() { // Given ArrayNode coreNode = nullChildNode(); - ArrayNode variantNode = ArrayNode.of(EncodingId.VORTEX_VARIANT, null, + ArrayNode variantNode = new ArrayNode(EncodingId.VORTEX_VARIANT, null, new ArrayNode[]{coreNode}, new int[]{}); ReadRegistry registry = TestRegistry.ofDecoders(SUT, new NullEncodingDecoder()); @@ -73,7 +73,7 @@ void decode_withShredded_decodesSecondChild() { ArrayNode coreNode = nullChildNode(); ArrayNode shreddedNode = primitiveChildNode(0); - ArrayNode variantNode = ArrayNode.of(EncodingId.VORTEX_VARIANT, meta, + ArrayNode variantNode = new ArrayNode(EncodingId.VORTEX_VARIANT, meta, new ArrayNode[]{coreNode, shreddedNode}, new int[]{}); MemorySegment[] segments = {TestSegments.leInts(1, 2, 3)}; @@ -96,7 +96,7 @@ void decode_withShredded_decodesSecondChild() { void decode_emptyMetadata_noShredded() { // Given ArrayNode coreNode = nullChildNode(); - ArrayNode variantNode = ArrayNode.of(EncodingId.VORTEX_VARIANT, MemorySegment.ofArray(new byte[0]), + ArrayNode variantNode = new ArrayNode(EncodingId.VORTEX_VARIANT, MemorySegment.ofArray(new byte[0]), new ArrayNode[]{coreNode}, new int[]{}); ReadRegistry registry = TestRegistry.ofDecoders(SUT, new NullEncodingDecoder()); @@ -116,7 +116,7 @@ void decode_nullableDtype_preservedOnResult() { // Given DType nullableVariant = new DType.Variant(true); ArrayNode coreNode = nullChildNode(); - ArrayNode variantNode = ArrayNode.of(EncodingId.VORTEX_VARIANT, null, + ArrayNode variantNode = new ArrayNode(EncodingId.VORTEX_VARIANT, null, new ArrayNode[]{coreNode}, new int[]{}); ReadRegistry registry = TestRegistry.ofDecoders(SUT, new NullEncodingDecoder()); @@ -134,7 +134,7 @@ void decode_nullableDtype_preservedOnResult() { @Test void decode_wrongChildCount_throws() { // Given - ArrayNode variantNode = ArrayNode.of(EncodingId.VORTEX_VARIANT, null, + ArrayNode variantNode = new ArrayNode(EncodingId.VORTEX_VARIANT, null, new ArrayNode[0], new int[]{}); ReadRegistry registry = TestRegistry.ofDecoders(SUT); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoderTest.java index 8519ad72..af4b5da5 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoderTest.java @@ -61,8 +61,8 @@ private static MemorySegment encodedLongs(long... signed) { private static Array decode(PType ptype, long n, MemorySegment encoded) { DType dtype = new DType.Primitive(ptype, false); - ArrayNode child = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ZIGZAG, null, new ArrayNode[]{child}, new int[]{}); + ArrayNode child = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ZIGZAG, null, new ArrayNode[]{child}, new int[]{}); DecodeContext ctx = new DecodeContext(node, dtype, n, new MemorySegment[]{encoded}, REGISTRY, Arena.ofAuto()); return SUT.decode(ctx); } @@ -202,7 +202,7 @@ void decode_i64_broadcastsSingleValue() { @Test void decode_nonPrimitiveDtype_throws() { // Given a non-primitive logical type on the context - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ZIGZAG, null, new ArrayNode[0], new int[]{}); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ZIGZAG, null, new ArrayNode[0], new int[]{}); DecodeContext ctx = new DecodeContext(node, DType.BOOL, 1, new MemorySegment[0], REGISTRY, Arena.ofAuto()); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/WriteRegistry.java b/writer/src/main/java/io/github/dfa1/vortex/writer/WriteRegistry.java index 847fd6f8..f6ea08fe 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/WriteRegistry.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/WriteRegistry.java @@ -88,7 +88,9 @@ public ExtensionEncoder lookup(ExtensionId extensionId) { /// Not thread-safe. Build once, use everywhere — the produced [WriteRegistry] is immutable. public static final class Builder { - private final Map encoders = new TreeMap<>(); + // EncodingId is not Comparable (a Custom id would throw on natural ordering); order by + // wire string, matching the constructor's sortedByName. + private final Map encoders = new TreeMap<>(Comparator.comparing(EncodingId::id)); private final Map extensions = new TreeMap<>(); private Builder() { diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/AlpEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/AlpEncodingEncoderTest.java index 57a0d58d..fd26b9d5 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/AlpEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/AlpEncodingEncoderTest.java @@ -54,7 +54,7 @@ private static DecodeContext buildAlpCtxF64( bb.putLong(v); } - ArrayNode encNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, + ArrayNode encNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); MemorySegment[] segments; @@ -71,8 +71,8 @@ private static DecodeContext buildAlpCtxF64( for (double v : patchValues) { vb.putDouble(v); } - ArrayNode idxNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); - ArrayNode valNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{2}); + ArrayNode idxNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); + ArrayNode valNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{2}); children = new ArrayNode[]{encNode, idxNode, valNode}; segments = new MemorySegment[]{ MemorySegment.ofArray(encBuf), MemorySegment.ofArray(idxBuf), MemorySegment.ofArray(valBuf)}; @@ -81,7 +81,7 @@ private static DecodeContext buildAlpCtxF64( segments = new MemorySegment[]{MemorySegment.ofArray(encBuf)}; } - ArrayNode alpNode = ArrayNode.of(EncodingId.VORTEX_ALP, + ArrayNode alpNode = new ArrayNode(EncodingId.VORTEX_ALP, MemorySegment.ofArray(metaBytes), children, new int[0]); return new DecodeContext(alpNode, DTypes.F64, encodedVals.length, segments, REGISTRY, java.lang.foreign.Arena.global()); @@ -94,8 +94,8 @@ private static DecodeContext buildAlpCtxF32(int expE, int expF, int[] encodedVal for (int v : encodedVals) { bb.putInt(v); } - ArrayNode encNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); - ArrayNode alpNode = ArrayNode.of(EncodingId.VORTEX_ALP, MemorySegment.ofArray(metaBytes), + ArrayNode encNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); + ArrayNode alpNode = new ArrayNode(EncodingId.VORTEX_ALP, MemorySegment.ofArray(metaBytes), new ArrayNode[]{encNode}, new int[0]); MemorySegment[] segments = {MemorySegment.ofArray(encBuf)}; return new DecodeContext(alpNode, DTypes.F32, encodedVals.length, segments, REGISTRY, java.lang.foreign.Arena.global()); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedConstantPatchesBroadcastTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedConstantPatchesBroadcastTest.java index 18eb4afa..c3a8cf0d 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedConstantPatchesBroadcastTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedConstantPatchesBroadcastTest.java @@ -51,11 +51,11 @@ void bitpackedDecode_withConstantPatchesValues_broadcastsValueAcrossPatches() { MemorySegment valBufSeg = arena.allocate(valScalarBytes.length, 1); MemorySegment.copy(MemorySegment.ofArray(valScalarBytes), 0, valBufSeg, 0, valScalarBytes.length); - ArrayNode idxChild = ArrayNode.of(EncodingId.VORTEX_CONSTANT, null, + ArrayNode idxChild = new ArrayNode(EncodingId.VORTEX_CONSTANT, null, new ArrayNode[0], new int[]{1}); - ArrayNode valChild = ArrayNode.of(EncodingId.VORTEX_CONSTANT, null, + ArrayNode valChild = new ArrayNode(EncodingId.VORTEX_CONSTANT, null, new ArrayNode[0], new int[]{2}); - ArrayNode root = ArrayNode.of(EncodingId.FASTLANES_BITPACKED, metaBuf, + ArrayNode root = new ArrayNode(EncodingId.FASTLANES_BITPACKED, metaBuf, new ArrayNode[]{idxChild, valChild}, new int[]{0}); DType dtype = DType.I64; diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingPatchesTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingPatchesTest.java index 3c643ca5..ed1438cd 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingPatchesTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingPatchesTest.java @@ -50,11 +50,11 @@ void decode_appliesPatches_overridingBitPackedValues() { byte[] valBuf = new byte[2 * 4]; ByteBuffer.wrap(valBuf).order(ByteOrder.LITTLE_ENDIAN).putInt(777).putInt(999); - ArrayNode idxNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, + ArrayNode idxNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); - ArrayNode valNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, + ArrayNode valNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{2}); - ArrayNode bpNode = ArrayNode.of(EncodingId.FASTLANES_BITPACKED, + ArrayNode bpNode = new ArrayNode(EncodingId.FASTLANES_BITPACKED, MemorySegment.ofArray(metaBytes), new ArrayNode[]{idxNode, valNode}, new int[]{0}); @@ -103,11 +103,11 @@ void encode_thenDecode_roundTripsWithPatches() { result.buffers().get(1), result.buffers().get(2) }; - ArrayNode idxNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, + ArrayNode idxNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); - ArrayNode valNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, + ArrayNode valNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{2}); - ArrayNode bpNode = ArrayNode.of(EncodingId.FASTLANES_BITPACKED, + ArrayNode bpNode = new ArrayNode(EncodingId.FASTLANES_BITPACKED, result.rootNode().metadata(), new ArrayNode[]{idxNode, valNode}, new int[]{0}); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ChunkedEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ChunkedEncodingEncoderTest.java index 20eb253a..98ada913 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ChunkedEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ChunkedEncodingEncoderTest.java @@ -34,7 +34,7 @@ private static ArrayNode toArrayNode(EncodeNode enc) { for (int i = 0; i < children.length; i++) { children[i] = toArrayNode(enc.children()[i]); } - return ArrayNode.of(enc.encodingId(), enc.metadata(), children, enc.bufferIndices()); + return new ArrayNode(enc.encodingId(), enc.metadata(), children, enc.bufferIndices()); } private static EncodeNode remapped(EncodeNode node, int offset) { @@ -115,7 +115,7 @@ void roundTrip_twoChunks_concatenatesValues() { ArrayNode offsetsNode = toArrayNode(offsetsResult.rootNode()); ArrayNode chunk0Node = toArrayNode(remapped(chunk0Result.rootNode(), 1)); ArrayNode chunk1Node = toArrayNode(remapped(chunk1Result.rootNode(), 2)); - ArrayNode root = ArrayNode.of( + ArrayNode root = new ArrayNode( EncodingId.VORTEX_CHUNKED, null, new ArrayNode[]{offsetsNode, chunk0Node, chunk1Node}, new int[]{}); @@ -150,7 +150,7 @@ void singleChunk_returnsSameValues() { chunkResult.buffers().getFirst() }; - ArrayNode root = ArrayNode.of( + ArrayNode root = new ArrayNode( EncodingId.VORTEX_CHUNKED, null, new ArrayNode[]{toArrayNode(offsetsResult.rootNode()), toArrayNode(remapped(chunkResult.rootNode(), 1))}, new int[]{}); @@ -172,7 +172,7 @@ void singleChunk_returnsSameValues() { void noChildren_throws() { // Given DType i64 = DType.I64; - ArrayNode root = ArrayNode.of(EncodingId.VORTEX_CHUNKED, null, new ArrayNode[]{}, new int[]{}); + ArrayNode root = new ArrayNode(EncodingId.VORTEX_CHUNKED, null, new ArrayNode[]{}, new int[]{}); DecodeContext ctx = new DecodeContext(root, i64, 0L, new MemorySegment[]{}, REGISTRY, Arena.ofAuto()); // When / Then diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoderTest.java index f6ac5ffa..0789a593 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoderTest.java @@ -44,7 +44,7 @@ private static ArrayNode toArrayNode(EncodeNode node) { for (int i = 0; i < children.length; i++) { children[i] = toArrayNode(node.children()[i]); } - return ArrayNode.of(node.encodingId(), node.metadata(), children, node.bufferIndices()); + return new ArrayNode(node.encodingId(), node.metadata(), children, node.bufferIndices()); } @Nested diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DecodeTestHelper.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DecodeTestHelper.java index d1afde8d..8240fdbe 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DecodeTestHelper.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DecodeTestHelper.java @@ -38,6 +38,6 @@ private static ArrayNode toArrayNode(EncodeNode enc) { for (int i = 0; i < children.length; i++) { children[i] = toArrayNode(enc.children()[i]); } - return ArrayNode.of(enc.encodingId(), enc.metadata(), children, enc.bufferIndices()); + return new ArrayNode(enc.encodingId(), enc.metadata(), children, enc.bufferIndices()); } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoderTest.java index 99a11cdf..90b78368 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ExtEncodingEncoderTest.java @@ -78,7 +78,7 @@ private ArrayNode encodeNodeToArrayNode(EncodeNode n) { for (int i = 0; i < children.length; i++) { children[i] = encodeNodeToArrayNode(n.children()[i]); } - return ArrayNode.of(n.encodingId(), n.metadata(), children, n.bufferIndices()); + return new ArrayNode(n.encodingId(), n.metadata(), children, n.bufferIndices()); } } @@ -130,8 +130,8 @@ void decode_extensionWrappingI64_returnsStorageArray() { DType storageDType = DType.I64; DType extDType = new DType.Extension("vortex.timestamp", storageDType, null, false); - ArrayNode primitiveNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); - ArrayNode extNode = ArrayNode.of(EncodingId.VORTEX_EXT, null, new ArrayNode[]{primitiveNode}, new int[0]); + ArrayNode primitiveNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); + ArrayNode extNode = new ArrayNode(EncodingId.VORTEX_EXT, null, new ArrayNode[]{primitiveNode}, new int[0]); DecodeContext ctx = new DecodeContext( extNode, extDType, values.length, new MemorySegment[]{buf}, REGISTRY, Arena.ofAuto()); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FixedSizeListEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FixedSizeListEncodingEncoderTest.java index 7eb4066a..e7d4f82f 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FixedSizeListEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FixedSizeListEncodingEncoderTest.java @@ -32,7 +32,7 @@ private static ArrayNode toArrayNode(EncodeNode node) { for (int i = 0; i < children.length; i++) { children[i] = toArrayNode(node.children()[i]); } - return ArrayNode.of(node.encodingId(), node.metadata(), children, node.bufferIndices()); + return new ArrayNode(node.encodingId(), node.metadata(), children, node.bufferIndices()); } @Nested @@ -123,7 +123,7 @@ void roundTrip_fixedSizeOne_preservesValues() { @Test void decode_wrongDtype_throws() { // Given - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_FIXED_SIZE_LIST, null, + ArrayNode node = new ArrayNode(EncodingId.VORTEX_FIXED_SIZE_LIST, null, new ArrayNode[0], new int[0]); DecodeContext ctx = new DecodeContext(node, DTypes.I32, 0, new MemorySegment[0], REGISTRY, Arena.global()); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoderTest.java index c9f62c0b..395bae15 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FrameOfReferenceEncodingEncoderTest.java @@ -55,9 +55,9 @@ private static DecodeContext buildForContext( } } - ArrayNode childNode = ArrayNode.of( + ArrayNode childNode = new ArrayNode( EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); - ArrayNode forNode = ArrayNode.of( + ArrayNode forNode = new ArrayNode( EncodingId.FASTLANES_FOR, MemorySegment.ofArray(metaBytes), new ArrayNode[]{childNode}, new int[0]); @@ -146,12 +146,12 @@ void decode_nullableResiduals_returnsMaskedArrayWithCorrectValues() { bb.putInt((int) v); } - ArrayNode validityNode = ArrayNode.of( + ArrayNode validityNode = new ArrayNode( EncodingId.VORTEX_BOOL, null, new ArrayNode[0], new int[]{1}); - ArrayNode primNode = ArrayNode.of( + ArrayNode primNode = new ArrayNode( EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[]{validityNode}, new int[]{0}); byte[] metaBytes = ProtoScalarValue.ofInt64Value(reference).encode(); - ArrayNode forNode = ArrayNode.of( + ArrayNode forNode = new ArrayNode( EncodingId.FASTLANES_FOR, MemorySegment.ofArray(metaBytes), new ArrayNode[]{primNode}, new int[0]); ReadRegistry registry = TestRegistry.ofDecoders( diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java index 1246e1cd..011d6721 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java @@ -43,7 +43,7 @@ private static ArrayNode toArrayNode(EncodeNode node) { for (int i = 0; i < children.length; i++) { children[i] = toArrayNode(node.children()[i]); } - return ArrayNode.of(node.encodingId(), node.metadata(), children, node.bufferIndices()); + return new ArrayNode(node.encodingId(), node.metadata(), children, node.bufferIndices()); } static Stream stringArrays() { @@ -156,11 +156,11 @@ private static DecodeContext buildCtx( byte[] metaBytes = new ProtoFSSTMetadata(io.github.dfa1.vortex.core.proto.ProtoPType.fromValue(PType.I32.ordinal()), io.github.dfa1.vortex.core.proto.ProtoPType.fromValue(PType.I32.ordinal())).encode(); - ArrayNode uncompLensNode = ArrayNode.of( + ArrayNode uncompLensNode = new ArrayNode( EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{3}); - ArrayNode codesOffNode = ArrayNode.of( + ArrayNode codesOffNode = new ArrayNode( EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{4}); - ArrayNode root = ArrayNode.of( + ArrayNode root = new ArrayNode( EncodingId.VORTEX_FSST, MemorySegment.ofArray(metaBytes), new ArrayNode[]{uncompLensNode, codesOffNode}, new int[]{0, 1, 2}); @@ -229,7 +229,7 @@ void decode_multipleStrings_decompressesAll() { @Test void decode_missingMetadata_throwsVortexException() { // Given - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_FSST, null, new ArrayNode[0], new int[0]); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_FSST, null, new ArrayNode[0], new int[0]); DecodeContext ctx = new DecodeContext(node, DTypes.UTF8, 0, new MemorySegment[0], REGISTRY, Arena.ofAuto()); // When diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ListEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ListEncodingEncoderTest.java index 67de3277..009cff3c 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ListEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ListEncodingEncoderTest.java @@ -33,7 +33,7 @@ private static ArrayNode toArrayNode(EncodeNode node) { for (int i = 0; i < children.length; i++) { children[i] = toArrayNode(node.children()[i]); } - return ArrayNode.of(node.encodingId(), node.metadata(), children, node.bufferIndices()); + return new ArrayNode(node.encodingId(), node.metadata(), children, node.bufferIndices()); } @Nested @@ -153,7 +153,7 @@ void roundTrip_singleList_preservesValues() { @Test void decode_wrongDtype_throws() { // Given - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_LIST, null, + ArrayNode node = new ArrayNode(EncodingId.VORTEX_LIST, null, new ArrayNode[0], new int[0]); DecodeContext ctx = new DecodeContext(node, DTypes.I32, 0, new MemorySegment[0], REGISTRY, Arena.global()); @@ -164,9 +164,9 @@ void decode_wrongDtype_throws() { @Test void decode_wrongChildCount_throws() { // Given - ArrayNode child = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, + ArrayNode child = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[0]); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_LIST, + ArrayNode node = new ArrayNode(EncodingId.VORTEX_LIST, MemorySegment.ofArray(new byte[0]), new ArrayNode[]{child}, new int[0]); DecodeContext ctx = new DecodeContext(node, DTypes.LIST_I32, 0, new MemorySegment[0], REGISTRY, Arena.global()); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ListViewEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ListViewEncodingEncoderTest.java index a304da71..8163e86e 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ListViewEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ListViewEncodingEncoderTest.java @@ -33,7 +33,7 @@ private static ArrayNode toArrayNode(EncodeNode node) { for (int i = 0; i < children.length; i++) { children[i] = toArrayNode(node.children()[i]); } - return ArrayNode.of(node.encodingId(), node.metadata(), children, node.bufferIndices()); + return new ArrayNode(node.encodingId(), node.metadata(), children, node.bufferIndices()); } @Nested @@ -149,7 +149,7 @@ void roundTrip_singleList_preservesValues() { @Test void decode_wrongDtype_throws() { // Given - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_LISTVIEW, null, + ArrayNode node = new ArrayNode(EncodingId.VORTEX_LISTVIEW, null, new ArrayNode[0], new int[0]); DecodeContext ctx = TestDecodeContexts.of(node, DTypes.I32).registry(REGISTRY).build(); @@ -160,9 +160,9 @@ void decode_wrongDtype_throws() { @Test void decode_wrongChildCount_throws() { // Given - ArrayNode child = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, + ArrayNode child = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[0]); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_LISTVIEW, + ArrayNode node = new ArrayNode(EncodingId.VORTEX_LISTVIEW, MemorySegment.ofArray(new byte[0]), new ArrayNode[]{child}, new int[0]); DecodeContext ctx = TestDecodeContexts.of(node, DTypes.LIST_I32).registry(REGISTRY).build(); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/NullEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/NullEncodingEncoderTest.java index a35a508e..fad2388b 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/NullEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/NullEncodingEncoderTest.java @@ -44,7 +44,7 @@ void encode_thenDecode_roundTrips() { // When EncodeResult resultEncoded = encoder.encode(DTypes.NULL, null, EncodeTestHelper.testCtx()); - ArrayNode node = ArrayNode.of(resultEncoded.rootNode().encodingId(), null, new ArrayNode[0], new int[0]); + ArrayNode node = new ArrayNode(resultEncoded.rootNode().encodingId(), null, new ArrayNode[0], new int[0]); DecodeContext ctx = new DecodeContext(node, DTypes.NULL, rowCount, new MemorySegment[0], ReadRegistry.empty(), Arena.ofAuto()); var result = decoder.decode(ctx); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PatchedEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PatchedEncodingEncoderTest.java index 0a6326b0..0926c202 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PatchedEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PatchedEncodingEncoderTest.java @@ -46,7 +46,7 @@ private static Array decode(EncodeResult result, DType dtype, int n) { ArrayNode idxNode = toArrayNode(enc[2]); ArrayNode valNode = toArrayNode(enc[3]); - ArrayNode patchedNode = ArrayNode.of(EncodingId.VORTEX_PATCHED, root.metadata(), + ArrayNode patchedNode = new ArrayNode(EncodingId.VORTEX_PATCHED, root.metadata(), new ArrayNode[]{innerNode, laneNode, idxNode, valNode}, new int[]{}); DecodeContext ctx = new DecodeContext(patchedNode, dtype, n, segments, REGISTRY, Arena.ofAuto()); @@ -54,7 +54,7 @@ private static Array decode(EncodeResult result, DType dtype, int n) { } private static ArrayNode toArrayNode(EncodeNode enc) { - return ArrayNode.of(enc.encodingId(), enc.metadata(), new ArrayNode[0], enc.bufferIndices()); + return new ArrayNode(enc.encodingId(), enc.metadata(), new ArrayNode[0], enc.bufferIndices()); } @Nested diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoderTest.java index a053a3a2..c22d1668 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoderTest.java @@ -143,9 +143,9 @@ void decode_withValidityChild_returnsMaskedArray() { MemorySegment valuesSeg = TestSegments.leInts(raw); MemorySegment validitySeg = MemorySegment.ofArray(new byte[]{0x05}); - ArrayNode validityNode = ArrayNode.of( + ArrayNode validityNode = new ArrayNode( EncodingId.VORTEX_BOOL, null, new ArrayNode[0], new int[]{1}); - ArrayNode primNode = ArrayNode.of( + ArrayNode primNode = new ArrayNode( EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[]{validityNode}, new int[]{0}); ReadRegistry registry = TestRegistry.ofDecoders(new PrimitiveEncodingDecoder(), new BoolEncodingDecoder()); @@ -178,7 +178,7 @@ void decode_noValidityChild_returnsPlainArray() { int[] raw = {1, 2, 3}; MemorySegment valuesSeg = TestSegments.leInts(raw); - ArrayNode primNode = ArrayNode.of( + ArrayNode primNode = new ArrayNode( EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); DType dtype = DType.I32; diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoderTest.java index bb3717db..d670f84d 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoderTest.java @@ -9,7 +9,6 @@ import io.github.dfa1.vortex.reader.decode.DecodeContext; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.reader.decode.KnownArrayNode; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; import io.github.dfa1.vortex.core.proto.ProtoRLEMetadata; @@ -34,15 +33,12 @@ class RleEncodingEncoderTest { private static final RleEncodingDecoder DECODER = new RleEncodingDecoder(); private static final ReadRegistry REGISTRY = TestRegistry.ofDecoders(DECODER, new PrimitiveEncodingDecoder()); - private static KnownArrayNode toArrayNode(EncodeNode enc) { + private static ArrayNode toArrayNode(EncodeNode enc) { ArrayNode[] children = new ArrayNode[enc.children().length]; for (int i = 0; i < children.length; i++) { children[i] = toArrayNode(enc.children()[i]); } - if (!(enc.encodingId() instanceof EncodingId.WellKnown wellKnown)) { - throw new IllegalStateException("expected well-known encoding id: " + enc.encodingId()); - } - return new KnownArrayNode(wellKnown, enc.metadata(), children, enc.bufferIndices()); + return new ArrayNode(enc.encodingId(), enc.metadata(), children, enc.bufferIndices()); } @Nested @@ -271,14 +267,14 @@ void decode_nullableIndices_returnsMaskedArrayWithCorrectValidity() { originalBufs.add(validityBuf); MemorySegment[] segments = originalBufs.toArray(new MemorySegment[0]); - KnownArrayNode origRoot = toArrayNode(encoded.rootNode()); - KnownArrayNode origIndices = (KnownArrayNode) origRoot.children()[1]; - ArrayNode validityNode = ArrayNode.of( + ArrayNode origRoot = toArrayNode(encoded.rootNode()); + ArrayNode origIndices = origRoot.children()[1]; + ArrayNode validityNode = new ArrayNode( EncodingId.VORTEX_BOOL, null, new ArrayNode[0], new int[]{3}); - ArrayNode nullableIndices = ArrayNode.of( + ArrayNode nullableIndices = new ArrayNode( origIndices.encodingId(), origIndices.metadata(), new ArrayNode[]{validityNode}, origIndices.bufferIndices()); - ArrayNode root = ArrayNode.of( + ArrayNode root = new ArrayNode( origRoot.encodingId(), origRoot.metadata(), new ArrayNode[]{origRoot.children()[0], nullableIndices, origRoot.children()[2]}, origRoot.bufferIndices()); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoderTest.java index d0566e11..b18effe4 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoderTest.java @@ -44,11 +44,11 @@ private static DecodeContext buildCtx( byte[] endsBuf = toLEBytes(ends, endsPtype); byte[] valBuf = toLEBytes(values, PType.I64); - ArrayNode endsNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, + ArrayNode endsNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); - ArrayNode valsNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, + ArrayNode valsNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); - ArrayNode reNode = ArrayNode.of(EncodingId.VORTEX_RUNEND, + ArrayNode reNode = new ArrayNode(EncodingId.VORTEX_RUNEND, MemorySegment.ofArray(metaBytes), new ArrayNode[]{endsNode, valsNode}, new int[0]); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java index e8d1f348..f7f2e7ec 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SequenceEncodingEncoderTest.java @@ -42,7 +42,7 @@ class Encode { private static DecodeContext encodeResultToCtx(EncodeResult result, DType dtype, long n) { MemorySegment meta = result.rootNode().metadata(); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_SEQUENCE, meta, new ArrayNode[0], new int[0]); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_SEQUENCE, meta, new ArrayNode[0], new int[0]); return new DecodeContext(node, dtype, n, new MemorySegment[0], ReadRegistry.empty(), Arena.ofAuto()); } @@ -314,7 +314,7 @@ static Stream i32Sequences() { } private static DecodeContext makeCtx(byte[] meta, DType dtype, long n) { - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_SEQUENCE, + ArrayNode node = new ArrayNode(EncodingId.VORTEX_SEQUENCE, MemorySegment.ofArray(meta), new ArrayNode[0], new int[0]); return new DecodeContext(node, dtype, n, new MemorySegment[0], ReadRegistry.empty(), Arena.ofAuto()); } @@ -401,7 +401,7 @@ void decode_emptySequence_returnsZeroLengthArray() { @Test void decode_missingMetadata_throwsVortexException() { - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_SEQUENCE, null, new ArrayNode[0], new int[0]); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_SEQUENCE, null, new ArrayNode[0], new int[0]); DecodeContext ctx = new DecodeContext(node, DTypes.I64, 3, new MemorySegment[0], ReadRegistry.empty(), Arena.ofAuto()); assertThatThrownBy(() -> DECODER.decode(ctx)).isInstanceOf(VortexException.class); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoderTest.java index 7baff9f3..13837dda 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/SparseEncodingEncoderTest.java @@ -52,11 +52,11 @@ private static Array decodeResult(EncodeResult encoded, DType dtype, int n) { MemorySegment[] segments = bufs.toArray(new MemorySegment[0]); - ArrayNode idxNode = ArrayNode.of(root.children()[0].encodingId(), null, + ArrayNode idxNode = new ArrayNode(root.children()[0].encodingId(), null, new ArrayNode[0], root.children()[0].bufferIndices()); - ArrayNode valNode = ArrayNode.of(root.children()[1].encodingId(), null, + ArrayNode valNode = new ArrayNode(root.children()[1].encodingId(), null, new ArrayNode[0], root.children()[1].bufferIndices()); - ArrayNode sparseNode = ArrayNode.of(root.encodingId(), root.metadata(), + ArrayNode sparseNode = new ArrayNode(root.encodingId(), root.metadata(), new ArrayNode[]{idxNode, valNode}, root.bufferIndices()); DecodeContext ctx = new DecodeContext(sparseNode, dtype, n, segments, REGISTRY, Arena.global()); @@ -175,11 +175,11 @@ private static DecodeContext buildSparseCtxF64( private static DecodeContext buildCtx(DType dtype, long rowCount, byte[] fillBytes, byte[] metaBytes, byte[] idxBuf, byte[] valBuf) { - ArrayNode idxNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, + ArrayNode idxNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); - ArrayNode valNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, + ArrayNode valNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{2}); - ArrayNode sparseNode = ArrayNode.of(EncodingId.VORTEX_SPARSE, + ArrayNode sparseNode = new ArrayNode(EncodingId.VORTEX_SPARSE, MemorySegment.ofArray(metaBytes), new ArrayNode[]{idxNode, valNode}, new int[]{0}); @@ -348,14 +348,14 @@ void decode_utf8_withPatches_writesStringsAtIndices() { byte[] offsets = intLEBytes(new int[]{0, 2, 5}); byte[] varBinMeta = new ProtoVarBinMetadata(io.github.dfa1.vortex.core.proto.ProtoPType.fromValue(PType.I32.ordinal())).encode(); - ArrayNode offsetsNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, + ArrayNode offsetsNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{3}); - ArrayNode valNode = ArrayNode.of(EncodingId.VORTEX_VARBIN, + ArrayNode valNode = new ArrayNode(EncodingId.VORTEX_VARBIN, MemorySegment.ofArray(varBinMeta), new ArrayNode[]{offsetsNode}, new int[]{2}); - ArrayNode idxNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, + ArrayNode idxNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); - ArrayNode sparseNode = ArrayNode.of(EncodingId.VORTEX_SPARSE, + ArrayNode sparseNode = new ArrayNode(EncodingId.VORTEX_SPARSE, MemorySegment.ofArray(meta), new ArrayNode[]{idxNode, valNode}, new int[]{0}); @@ -391,11 +391,11 @@ void decode_bool_withPatches_setsBitsAtIndices() { byte[] idxBuf = toLEBytes(new long[]{2L, 5L}, PType.U32); byte[] boolBits = new byte[]{0b00000011}; - ArrayNode valNode = ArrayNode.of(EncodingId.VORTEX_BOOL, null, + ArrayNode valNode = new ArrayNode(EncodingId.VORTEX_BOOL, null, new ArrayNode[0], new int[]{2}); - ArrayNode idxNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, + ArrayNode idxNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{1}); - ArrayNode sparseNode = ArrayNode.of(EncodingId.VORTEX_SPARSE, + ArrayNode sparseNode = new ArrayNode(EncodingId.VORTEX_SPARSE, MemorySegment.ofArray(meta), new ArrayNode[]{idxNode, valNode}, new int[]{0}); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/StructEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/StructEncodingEncoderTest.java index 9f9ab1f3..7d77c323 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/StructEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/StructEncodingEncoderTest.java @@ -37,7 +37,7 @@ private static ArrayNode toArrayNode(EncodeNode node) { for (int i = 0; i < children.length; i++) { children[i] = toArrayNode(node.children()[i]); } - return ArrayNode.of(node.encodingId(), node.metadata(), children, node.bufferIndices()); + return new ArrayNode(node.encodingId(), node.metadata(), children, node.bufferIndices()); } @Nested @@ -115,12 +115,12 @@ void fieldCountMismatch_throwsVortexException() { class Decode { private static ArrayNode primitiveNode(int bufferIdx) { - return ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], + return new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{bufferIdx}); } private static ArrayNode boolNode(int bufferIdx) { - return ArrayNode.of(EncodingId.VORTEX_BOOL, null, new ArrayNode[0], + return new ArrayNode(EncodingId.VORTEX_BOOL, null, new ArrayNode[0], new int[]{bufferIdx}); } @@ -130,7 +130,7 @@ void decode_nonNullableWrapper_oneChild_returnsValues() { long[] data = {10L, 20L, 30L}; MemorySegment seg = TestSegments.leLongs(data); ArrayNode valuesNode = primitiveNode(0); - ArrayNode structNode = ArrayNode.of(EncodingId.VORTEX_STRUCT, null, + ArrayNode structNode = new ArrayNode(EncodingId.VORTEX_STRUCT, null, new ArrayNode[]{valuesNode}, new int[0]); DecodeContext ctx = new DecodeContext(structNode, DTypes.I64, data.length, new MemorySegment[]{seg}, REGISTRY, Arena.global()); @@ -154,7 +154,7 @@ void decode_nullableWrapper_twoChildren_returnsMaskedArray() { ArrayNode validityNode = boolNode(0); ArrayNode valuesNode = primitiveNode(1); - ArrayNode structNode = ArrayNode.of(EncodingId.VORTEX_STRUCT, null, + ArrayNode structNode = new ArrayNode(EncodingId.VORTEX_STRUCT, null, new ArrayNode[]{validityNode, valuesNode}, new int[0]); ReadRegistry registry = TestRegistry.ofDecoders(DECODER, new PrimitiveEncodingDecoder(), new BoolEncodingDecoder()); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoderTest.java index 66811f87..d7f2d257 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoderTest.java @@ -159,7 +159,7 @@ class Decode { @Test void decode_missingMetadata_throwsVortexException() { // Given - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_VARBIN, null, new ArrayNode[0], new int[0]); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_VARBIN, null, new ArrayNode[0], new int[0]); DecodeContext ctx = new DecodeContext(node, DTypes.UTF8, 3, new MemorySegment[0], ReadRegistry.empty(), Arena.ofAuto()); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoderTest.java index 9a99a453..65fe5807 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoderTest.java @@ -71,7 +71,7 @@ void encode_thenDecode_roundtripsAllStrings(String name, String[] values) { // When EncodeResult result = ENCODER.encode(DTypes.UTF8, values, EncodeTestHelper.testCtx()); MemorySegment[] bufs = result.buffers().toArray(MemorySegment[]::new); - ArrayNode node = ArrayNode.of( + ArrayNode node = new ArrayNode( EncodingId.VORTEX_VARBINVIEW, null, new ArrayNode[0], result.rootNode().bufferIndices()); DecodeContext ctx = new DecodeContext(node, DTypes.UTF8, values.length, bufs, REGISTRY, arena); @@ -154,7 +154,7 @@ void decode_roundtrip_returnsAllStrings(String name, String[] values) { segBufs = new MemorySegment[]{views}; } - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_VARBINVIEW, null, + ArrayNode node = new ArrayNode(EncodingId.VORTEX_VARBINVIEW, null, new ArrayNode[0], bufIndices); DecodeContext ctx = new DecodeContext(node, DTypes.UTF8, n, segBufs, REGISTRY, arena); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VariantEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VariantEncodingEncoderTest.java index c378dee5..f071729d 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VariantEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VariantEncodingEncoderTest.java @@ -157,7 +157,7 @@ private static io.github.dfa1.vortex.reader.decode.ArrayNode toArrayNode(EncodeN for (int i = 0; i < children.length; i++) { children[i] = toArrayNode(node.children()[i]); } - return io.github.dfa1.vortex.reader.decode.ArrayNode.of( + return new io.github.dfa1.vortex.reader.decode.ArrayNode( node.encodingId(), node.metadata(), children, node.bufferIndices()); } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoderTest.java index d87a4b0a..fda0f718 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoderTest.java @@ -55,8 +55,8 @@ private static DecodeContext buildI32Ctx(int[] encodedUnsigned) { buf.flip(); MemorySegment seg = MemorySegment.ofBuffer(buf); - ArrayNode primitiveNode = ArrayNode.of(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); - ArrayNode zigzagNode = ArrayNode.of(EncodingId.VORTEX_ZIGZAG, null, new ArrayNode[]{primitiveNode}, new int[0]); + ArrayNode primitiveNode = new ArrayNode(EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{0}); + ArrayNode zigzagNode = new ArrayNode(EncodingId.VORTEX_ZIGZAG, null, new ArrayNode[]{primitiveNode}, new int[0]); return new DecodeContext(zigzagNode, DTypes.I32, encodedUnsigned.length, new MemorySegment[]{seg}, REGISTRY, Arena.ofAuto()); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoderTest.java index d4fd08a5..2dced95c 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoderTest.java @@ -218,7 +218,7 @@ private static DecodeContext makeDictCtx( segments[i + 1] = MemorySegment.ofArray(compressedFrames[i]); bufIndices[i + 1] = i + 1; } - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ZSTD, MemorySegment.ofArray(meta), + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ZSTD, MemorySegment.ofArray(meta), new ArrayNode[0], bufIndices); return new DecodeContext(node, dtype, n, segments, ReadRegistry.empty(), Arena.ofAuto()); } @@ -239,7 +239,7 @@ private static DecodeContext makeNullableCtx( allSegments.addAll(validityResult.buffers()); ArrayNode validityNode = toArrayNode(remappedValidity); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ZSTD, MemorySegment.ofArray(meta), + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ZSTD, MemorySegment.ofArray(meta), new ArrayNode[]{validityNode}, bufIndices); ReadRegistry registry = TestRegistry.ofDecoders(new BoolEncodingDecoder()); @@ -253,7 +253,7 @@ private static ArrayNode toArrayNode(EncodeNode enc) { for (int i = 0; i < children.length; i++) { children[i] = toArrayNode(enc.children()[i]); } - return ArrayNode.of(enc.encodingId(), enc.metadata(), children, enc.bufferIndices()); + return new ArrayNode(enc.encodingId(), enc.metadata(), children, enc.bufferIndices()); } private static byte[] compress(byte[] input) { @@ -401,7 +401,7 @@ void decode_allNull_returnsEmptyMaskedArray() { @Test void decode_missingMetadata_throwsVortexException() { - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ZSTD, null, new ArrayNode[0], new int[0]); + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ZSTD, null, new ArrayNode[0], new int[0]); DecodeContext ctx = new DecodeContext(node, DTypes.I32, 0, new MemorySegment[0], ReadRegistry.empty(), Arena.ofAuto()); @@ -417,7 +417,7 @@ void decode_negativeFrameSize_throwsVortexException() { // IoBounds.toIntSize guard must convert it to a VortexException first. byte[] compressed = compress(toLeBytes(new int[]{0})); byte[] meta = metaNoDict(new long[]{-1}, new long[]{1}); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ZSTD, MemorySegment.ofArray(meta), + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ZSTD, MemorySegment.ofArray(meta), new ArrayNode[0], new int[]{0}); DecodeContext ctx = new DecodeContext(node, DTypes.I32, 1, new MemorySegment[]{MemorySegment.ofArray(compressed)}, ReadRegistry.empty(), Arena.ofAuto()); @@ -434,7 +434,7 @@ void decode_oversizedFrameSize_throwsVortexException() { // asSlice site in decompressFrames. byte[] compressed = compress(toLeBytes(new int[]{0})); byte[] meta = metaNoDict(new long[]{(long) Integer.MAX_VALUE + 1}, new long[]{1}); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ZSTD, MemorySegment.ofArray(meta), + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ZSTD, MemorySegment.ofArray(meta), new ArrayNode[0], new int[]{0}); DecodeContext ctx = new DecodeContext(node, DTypes.I32, 1, new MemorySegment[]{MemorySegment.ofArray(compressed)}, ReadRegistry.empty(), Arena.ofAuto()); @@ -454,7 +454,7 @@ void decode_varBinOversizedLengthPrefix_throwsVortexException() { byte[] raw = toLeBytes(new int[]{1_000_000}); byte[] compressed = compress(raw); byte[] meta = metaNoDict(new long[]{raw.length}, new long[]{1}); - ArrayNode node = ArrayNode.of(EncodingId.VORTEX_ZSTD, MemorySegment.ofArray(meta), + ArrayNode node = new ArrayNode(EncodingId.VORTEX_ZSTD, MemorySegment.ofArray(meta), new ArrayNode[0], new int[]{0}); DecodeContext ctx = new DecodeContext(node, new DType.Utf8(false), 1, new MemorySegment[]{MemorySegment.ofArray(compressed)}, ReadRegistry.empty(), Arena.ofAuto()); From 6f7952e93e4ed4e8fa2531023a37190883d7426a Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 4 Jul 2026 11:07:20 +0200 Subject: [PATCH 6/7] docs(changelog): sealed EncodingId + typed ArrayNode Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5718ba5d..b26b9cfb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `Compute.filteredSum` over a dictionary-encoded filter column is ~20× faster (best runs ~30×): the predicate is resolved against the dictionary's value pool once and the raw `u8` codes are scanned directly from their backing segments, instead of decoding every row through the per-element accessor — a fused `SUM(measure) WHERE category = …` over 100M rows drops from ~760 ms to ~38 ms. ([85e251cc](https://github.com/dfa1/vortex-java/commit/85e251cc)) - `Compute.filteredAggregate` takes the same dictionary code-scan lane (`COUNT(*)` included) — ~22× faster on the same workload (~980 ms → ~46 ms over 100M rows), which the Calcite `WHERE`-filtered aggregate push-down inherits on its boundary chunks. ([145791c7](https://github.com/dfa1/vortex-java/commit/145791c7), [6e6d7dd0](https://github.com/dfa1/vortex-java/commit/6e6d7dd0)) - A multi-column `AND` filter no longer forfeits the dictionary lane: the dict-encoded leaf drives the code scan and the remaining predicates are evaluated only on its matches — `SUM(…) WHERE category = 7 AND price > 500` over 100M rows drops from ~2.3 s to ~200 ms (~11×). ([12e13501](https://github.com/dfa1/vortex-java/commit/12e13501)) +- `core.model.EncodingId` is now a sealed interface: the spec constants live in the nested `WellKnown` enum (re-exported, so `EncodingId.VORTEX_FOO` call sites compile unchanged) and `Custom` wraps any other wire string, which for the first time lets third-party `EncodingDecoder`/`EncodingEncoder` implementations declare ids outside the spec set. `parse` is total over non-blank ids — an unknown id yields a typed `Custom` instead of an empty `Optional`. ([ea88a91b](https://github.com/dfa1/vortex-java/commit/ea88a91b)) +- `reader.decode.ArrayNode` is a single record carrying the typed `EncodingId`; the `KnownArrayNode`/`UnknownArrayNode` split and the `ArrayNode.of` factory are gone. Decode dispatch, the `allowUnknown` passthrough, and error messages are unchanged. A crafted file with a blank encoding id now fails as `VortexException` instead of escaping as `IllegalArgumentException`. ([21810d7e](https://github.com/dfa1/vortex-java/commit/21810d7e)) ## [0.11.0] — 2026-06-28 From 3c412b40ae76902efc207d9b20bcfa65e3a88da1 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sat, 4 Jul 2026 11:11:16 +0200 Subject: [PATCH 7/7] docs(core): vortex.flat is a layout id, not an encoding id The EncodingId javadoc examples used "vortex.flat", which lives in the footer's layoutSpecs table (Layout.FLAT), never in arraySpecs; the canonical flat encoding id is "vortex.primitive". Co-Authored-By: Claude Fable 5 --- .../java/io/github/dfa1/vortex/core/model/EncodingId.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/github/dfa1/vortex/core/model/EncodingId.java b/core/src/main/java/io/github/dfa1/vortex/core/model/EncodingId.java index 6f5c1d90..84ed2131 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/model/EncodingId.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/model/EncodingId.java @@ -9,7 +9,7 @@ /// Identity of an array encoding — either a spec [WellKnown] constant or a third-party [Custom] id. /// -/// The wire representation is always a string (e.g. `"vortex.flat"`); [#parse(String)] maps any +/// The wire representation is always a string (e.g. `"vortex.primitive"`); [#parse(String)] maps any /// such string to a typed value, and [#id()] recovers the wire string from a typed value. /// /// Extends [Serializable] so a [Custom] or [WellKnown] carried on a @@ -17,7 +17,7 @@ /// enum's implicit serializability. public sealed interface EncodingId extends Serializable permits EncodingId.WellKnown, EncodingId.Custom { - /// Returns the wire string of this encoding id (e.g. `"vortex.flat"`). + /// Returns the wire string of this encoding id (e.g. `"vortex.primitive"`). /// /// @return the wire string of this encoding id String id();