diff --git a/CHANGELOG.md b/CHANGELOG.md index f6d4a7fa..7154f520 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- The little-endian `ValueLayout` constants moved from `PTypeIO.LE_*` to `VortexFormat.LE_*` — endianness is a property of the wire format, not of ptypes, and `VortexFormat` is where format facts live. Six classes that carried private copies (including reversed-name duplicates like `SHORT_LE`) now share the single source; nothing outside `VortexFormat` defines a `withOrder(LITTLE_ENDIAN)` layout. ([c060d34f](https://github.com/dfa1/vortex-java/commit/c060d34f)) +- `Chunk.columns()` returns an order-preserving `SequencedMap` — one map instead of two parallel string-keyed maps, with each column's `Array` and `DType` traveling together in the `Column` carrier. `column(String)` stays as boundary sugar (plus a `column(ColumnName)` overload); iteration order is the schema/projection order, now guaranteed even for 1–2 column chunks. Typed names originate in `ScanIterator` from the file's already-certified schema. ([f8ad15d1](https://github.com/dfa1/vortex-java/commit/f8ad15d1)) + - `Compute.filteredSum` over a dictionary-encoded filter column is ~20× faster (best runs ~30×): the predicate is resolved against the dictionary's value pool once and the raw `u8` codes are scanned directly from their backing segments, instead of decoding every row through the per-element accessor — a fused `SUM(measure) WHERE category = …` over 100M rows drops from ~760 ms to ~38 ms. ([85e251cc](https://github.com/dfa1/vortex-java/commit/85e251cc)) - `Compute.filteredAggregate` takes the same dictionary code-scan lane (`COUNT(*)` included) — ~22× faster on the same workload (~980 ms → ~46 ms over 100M rows), which the Calcite `WHERE`-filtered aggregate push-down inherits on its boundary chunks. ([145791c7](https://github.com/dfa1/vortex-java/commit/145791c7), [6e6d7dd0](https://github.com/dfa1/vortex-java/commit/6e6d7dd0)) - A multi-column `AND` filter no longer forfeits the dictionary lane: the dict-encoded leaf drives the code scan and the remaining predicates are evaluated only on its matches — `SUM(…) WHERE category = 7 AND price > 500` over 100M rows drops from ~2.3 s to ~200 ms (~11×). ([12e13501](https://github.com/dfa1/vortex-java/commit/12e13501)) diff --git a/CLAUDE.md b/CLAUDE.md index 2c9d6c11..18263c36 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -206,6 +206,15 @@ in the Rust source for the exact schema, then implement from spec. - **POM deps** grouped with comments: `` then ``, each with project-internal (`io.github.dfa1.vortex:*`) deps first, then external. Omit empty sections. +## Documentation is part of every change + +Living docs ship in the same commit/PR as the change they describe — never as a follow-up +sweep. A change touching public API, module structure, wire behavior, or policy updates +whichever apply: `docs/reference.md`, `docs/compatibility.md`, the CLAUDE.md module map / +design decisions, and CHANGELOG (per its own rules). Historical records (`adr/`, released +CHANGELOG sections) are exempt — they describe the past. Docs drift is a bug (2026-07-04: a +single audit found phantom APIs, dead service files, and pre-refactor FQNs across four files). + ## Code style - 4-space indent, **zero SonarQube bugs/smells**, no `sun.misc.Unsafe` or internal JDK APIs. diff --git a/cli/src/main/java/io/github/dfa1/vortex/cli/tui/VortexInspectorTui.java b/cli/src/main/java/io/github/dfa1/vortex/cli/tui/VortexInspectorTui.java index fe011c2c..88455491 100644 --- a/cli/src/main/java/io/github/dfa1/vortex/cli/tui/VortexInspectorTui.java +++ b/cli/src/main/java/io/github/dfa1/vortex/cli/tui/VortexInspectorTui.java @@ -3,6 +3,7 @@ import io.github.dfa1.vortex.reader.layout.Layout; import io.github.dfa1.vortex.reader.SegmentSpec; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.cli.tui.term.Ansi; import io.github.dfa1.vortex.cli.tui.term.Key; @@ -766,11 +767,12 @@ private void runDataLoad(String columnName) { return; } try (Chunk chunk = it.next()) { - Array array = chunk.columns().get(columnName); - if (array == null) { + Chunk.Column column = chunk.columns().get(ColumnName.of(columnName)); + if (column == null) { dataCache.put(columnName, new DataState.Loaded(List.of())); return; } + Array array = column.array(); int n = (int) Math.min(array.length(), DATA_PREVIEW_ROWS); List out = new ArrayList<>(n); for (int i = 0; i < n; i++) { diff --git a/core/src/main/java/io/github/dfa1/vortex/core/compute/PrimitiveArrays.java b/core/src/main/java/io/github/dfa1/vortex/core/compute/PrimitiveArrays.java index 25d68e4d..1091443e 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/compute/PrimitiveArrays.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/compute/PrimitiveArrays.java @@ -2,6 +2,7 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.io.PTypeIO; import io.github.dfa1.vortex.core.error.VortexException; @@ -97,7 +98,7 @@ public static long[] toLongs(Object data, PType ptype, EncodingId encoding) { public static MemorySegment fromLongs(long[] longs, PType ptype, SegmentAllocator arena) { if (ptype == PType.I64 || ptype == PType.U64) { MemorySegment dst = arena.allocate((long) longs.length * 8); - MemorySegment.copy(MemorySegment.ofArray(longs), ValueLayout.JAVA_LONG, 0L, dst, PTypeIO.LE_LONG, 0L, longs.length); + MemorySegment.copy(MemorySegment.ofArray(longs), ValueLayout.JAVA_LONG, 0L, dst, VortexFormat.LE_LONG, 0L, longs.length); return dst; } int n = longs.length; diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsBuilder.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsBuilder.java index 63ef8f44..58d231a7 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsBuilder.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsBuilder.java @@ -1,10 +1,10 @@ package io.github.dfa1.vortex.core.fbs; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_DOUBLE; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_FLOAT; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_INT; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_LONG; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_SHORT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_DOUBLE; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_FLOAT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_LONG; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_SHORT; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsStruct.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsStruct.java index 97239e2c..3498c111 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsStruct.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsStruct.java @@ -1,10 +1,10 @@ package io.github.dfa1.vortex.core.fbs; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_DOUBLE; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_FLOAT; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_INT; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_LONG; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_SHORT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_DOUBLE; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_FLOAT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_LONG; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_SHORT; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; diff --git a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsTable.java b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsTable.java index 8f98b956..544e5ce0 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsTable.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/fbs/FbsTable.java @@ -1,10 +1,10 @@ package io.github.dfa1.vortex.core.fbs; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_DOUBLE; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_FLOAT; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_INT; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_LONG; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_SHORT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_DOUBLE; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_FLOAT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_LONG; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_SHORT; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; diff --git a/core/src/main/java/io/github/dfa1/vortex/core/io/PTypeIO.java b/core/src/main/java/io/github/dfa1/vortex/core/io/PTypeIO.java index 4ec48301..a57a6d8b 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/io/PTypeIO.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/io/PTypeIO.java @@ -9,7 +9,12 @@ import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodType; import java.lang.invoke.VarHandle; -import java.nio.ByteOrder; + +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_DOUBLE; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_FLOAT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_LONG; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_SHORT; /// Bulk I/O helpers for primitive ptypes, backed by `MemorySegment`/`ValueLayout`/`MethodHandle`. /// @@ -18,21 +23,10 @@ /// the unaligned little-endian VarHandle. This lets hot loops avoid per-element `switch` /// dispatch on `PType`. /// -/// The LE_* layout constants are public so callers outside this package can share them -/// without duplicating the `withOrder(LITTLE_ENDIAN)` boilerplate. +/// The shared little-endian layouts live in [VortexFormat] — endianness is a property of the +/// wire format, not of ptypes; this class only maps ptypes onto those layouts. public final class PTypeIO { - /// Unaligned little-endian layout for 16-bit shorts. - public static final ValueLayout.OfShort LE_SHORT = ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); - /// Unaligned little-endian layout for 32-bit ints. - public static final ValueLayout.OfInt LE_INT = ValueLayout.JAVA_INT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); - /// Unaligned little-endian layout for 64-bit longs. - public static final ValueLayout.OfLong LE_LONG = ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); - /// Unaligned little-endian layout for 32-bit floats. - public static final ValueLayout.OfFloat LE_FLOAT = ValueLayout.JAVA_FLOAT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); - /// Unaligned little-endian layout for 64-bit doubles. - public static final ValueLayout.OfDouble LE_DOUBLE = ValueLayout.JAVA_DOUBLE_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); - private static final MethodHandle[] SETTERS = buildSetters(); private PTypeIO() { diff --git a/core/src/main/java/io/github/dfa1/vortex/core/io/VortexFormat.java b/core/src/main/java/io/github/dfa1/vortex/core/io/VortexFormat.java index 3df9bf68..186ce839 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/io/VortexFormat.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/io/VortexFormat.java @@ -1,6 +1,9 @@ package io.github.dfa1.vortex.core.io; import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +import java.nio.ByteOrder; /// Wire-format constants for the Vortex file format. /// @@ -24,6 +27,22 @@ public final class VortexFormat { /// Files with any other version are rejected up front rather than silently mis-parsed. public static final int VERSION = 1; + // All multi-byte integers in the Vortex wire format are little-endian — trailer fields, + // spec-table indexes, buffer scaffolding, and element values alike. These unaligned + // little-endian layouts are the single source for every wire read/write; nothing outside + // this class defines its own withOrder(LITTLE_ENDIAN) copy. + + /// Unaligned little-endian layout for 16-bit shorts. + public static final ValueLayout.OfShort LE_SHORT = ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); + /// Unaligned little-endian layout for 32-bit ints. + public static final ValueLayout.OfInt LE_INT = ValueLayout.JAVA_INT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); + /// Unaligned little-endian layout for 64-bit longs. + public static final ValueLayout.OfLong LE_LONG = ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); + /// Unaligned little-endian layout for 32-bit floats. + public static final ValueLayout.OfFloat LE_FLOAT = ValueLayout.JAVA_FLOAT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); + /// Unaligned little-endian layout for 64-bit doubles. + public static final ValueLayout.OfDouble LE_DOUBLE = ValueLayout.JAVA_DOUBLE_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); + private VortexFormat() { } } diff --git a/core/src/main/java/io/github/dfa1/vortex/core/model/TimestampDtype.java b/core/src/main/java/io/github/dfa1/vortex/core/model/TimestampDtype.java index 054f5177..3d198385 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/model/TimestampDtype.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/model/TimestampDtype.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.core.error.VortexException; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_SHORT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_SHORT; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; diff --git a/core/src/main/java/io/github/dfa1/vortex/core/proto/ProtoReader.java b/core/src/main/java/io/github/dfa1/vortex/core/proto/ProtoReader.java index 2cfd6129..a95d2ffd 100644 --- a/core/src/main/java/io/github/dfa1/vortex/core/proto/ProtoReader.java +++ b/core/src/main/java/io/github/dfa1/vortex/core/proto/ProtoReader.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.core.proto; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_INT; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_LONG; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_LONG; import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; diff --git a/core/src/test/java/io/github/dfa1/vortex/core/compute/PrimitiveArraysTest.java b/core/src/test/java/io/github/dfa1/vortex/core/compute/PrimitiveArraysTest.java index 26fd260b..aef3128e 100644 --- a/core/src/test/java/io/github/dfa1/vortex/core/compute/PrimitiveArraysTest.java +++ b/core/src/test/java/io/github/dfa1/vortex/core/compute/PrimitiveArraysTest.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.core.compute; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; @@ -142,7 +142,7 @@ void fromLongs_i64_writesLittleEndian() { // Then it is stored little-endian (lowest byte first) assertThat(seg.get(ValueLayout.JAVA_BYTE, 0)).isEqualTo((byte) 0x08); - assertThat(seg.getAtIndex(PTypeIO.LE_LONG, 0)).isEqualTo(0x0102_0304_0506_0708L); + assertThat(seg.getAtIndex(VortexFormat.LE_LONG, 0)).isEqualTo(0x0102_0304_0506_0708L); } } @@ -164,9 +164,9 @@ void fromLongs_narrowWidth_keepsOnlyLowBytes() { private static long readElement(MemorySegment seg, PType ptype, int i) { return switch (ptype) { case I8, U8 -> seg.get(ValueLayout.JAVA_BYTE, i); - case I16, U16 -> seg.getAtIndex(PTypeIO.LE_SHORT, i); - case I32, U32 -> seg.getAtIndex(PTypeIO.LE_INT, i); - case I64, U64 -> seg.getAtIndex(PTypeIO.LE_LONG, i); + case I16, U16 -> seg.getAtIndex(VortexFormat.LE_SHORT, i); + case I32, U32 -> seg.getAtIndex(VortexFormat.LE_INT, i); + case I64, U64 -> seg.getAtIndex(VortexFormat.LE_LONG, i); default -> throw new IllegalArgumentException("not an integer ptype: " + ptype); }; } diff --git a/core/src/test/java/io/github/dfa1/vortex/core/io/PTypeIOTest.java b/core/src/test/java/io/github/dfa1/vortex/core/io/PTypeIOTest.java index c21f584f..e2010adc 100644 --- a/core/src/test/java/io/github/dfa1/vortex/core/io/PTypeIOTest.java +++ b/core/src/test/java/io/github/dfa1/vortex/core/io/PTypeIOTest.java @@ -5,11 +5,11 @@ import java.lang.foreign.MemorySegment; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_DOUBLE; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_FLOAT; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_INT; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_LONG; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_SHORT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_DOUBLE; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_FLOAT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_LONG; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_SHORT; import static java.lang.foreign.ValueLayout.JAVA_BYTE; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; diff --git a/core/src/test/java/io/github/dfa1/vortex/encoding/TestSegments.java b/core/src/test/java/io/github/dfa1/vortex/encoding/TestSegments.java index 5e64f08d..771128b9 100644 --- a/core/src/test/java/io/github/dfa1/vortex/encoding/TestSegments.java +++ b/core/src/test/java/io/github/dfa1/vortex/encoding/TestSegments.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.encoding; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; @@ -16,7 +16,7 @@ private TestSegments() { public static MemorySegment leLongs(long... values) { MemorySegment seg = Arena.ofAuto().allocate((long) values.length * Long.BYTES); for (int i = 0; i < values.length; i++) { - seg.setAtIndex(PTypeIO.LE_LONG, i, values[i]); + seg.setAtIndex(VortexFormat.LE_LONG, i, values[i]); } return seg; } @@ -24,7 +24,7 @@ public static MemorySegment leLongs(long... values) { public static MemorySegment leInts(int... values) { MemorySegment seg = Arena.ofAuto().allocate((long) values.length * Integer.BYTES); for (int i = 0; i < values.length; i++) { - seg.setAtIndex(PTypeIO.LE_INT, i, values[i]); + seg.setAtIndex(VortexFormat.LE_INT, i, values[i]); } return seg; } @@ -32,7 +32,7 @@ public static MemorySegment leInts(int... values) { public static MemorySegment leDoubles(double... values) { MemorySegment seg = Arena.ofAuto().allocate((long) values.length * Double.BYTES); for (int i = 0; i < values.length; i++) { - seg.setAtIndex(PTypeIO.LE_DOUBLE, i, values[i]); + seg.setAtIndex(VortexFormat.LE_DOUBLE, i, values[i]); } return seg; } @@ -40,7 +40,7 @@ public static MemorySegment leDoubles(double... values) { public static MemorySegment leFloats(float... values) { MemorySegment seg = Arena.ofAuto().allocate((long) values.length * Float.BYTES); for (int i = 0; i < values.length; i++) { - seg.setAtIndex(PTypeIO.LE_FLOAT, i, values[i]); + seg.setAtIndex(VortexFormat.LE_FLOAT, i, values[i]); } return seg; } @@ -48,7 +48,7 @@ public static MemorySegment leFloats(float... values) { public static MemorySegment leShorts(short... values) { MemorySegment seg = Arena.ofAuto().allocate((long) values.length * Short.BYTES); for (int i = 0; i < values.length; i++) { - seg.setAtIndex(PTypeIO.LE_SHORT, i, values[i]); + seg.setAtIndex(VortexFormat.LE_SHORT, i, values[i]); } return seg; } diff --git a/docs/reference.md b/docs/reference.md index d2203a8a..a2b555ad 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -183,13 +183,20 @@ columnar buffers; closing the chunk releases the arena. After `close()`, touchin any `Array` previously returned by `column(...)` or `columns()` raises FFM's scope check (`IllegalStateException`). -| Method | Notes | -|-----------------------------------------|----------------------------------------------------------| -| `rowCount()` | Rows in this chunk | -| `columns()` | All columns in this chunk | -| ` column(String name)` | Typed column lookup; throws `VortexException` if unknown | -| `isClosed()` | Whether `close()` has run | -| `close()` | Releases the chunk's arena. Idempotent. | +Columns are stored as one order-preserving map keyed by the validated [`ColumnName`]; each +entry is a `Chunk.Column(Array array, DType dtype)` carrier, so a column's data and type can +never desync. `column(String)` is boundary sugar: the name is wrapped in a `ColumnName` (a +policy-invalid name fails fast — it could never match a certified column). + +| Method | Notes | +|---------------------------------------------|----------------------------------------------------------| +| `rowCount()` | Rows in this chunk | +| `columns()` | `SequencedMap`, schema order, unmodifiable | +| ` column(String name)` | Typed column lookup; throws `VortexException` if absent | +| ` column(ColumnName name)` | Same, for callers that validated early | +| `as(String name, Class domainType)` | Extension column → typed `List` | +| `isClosed()` | Whether `close()` has run | +| `close()` | Releases the chunk's arena. Idempotent. | --- diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java index 2e561cc9..82bfbf7b 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.inspect; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_INT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; import io.github.dfa1.vortex.reader.ArrayStats; import io.github.dfa1.vortex.core.model.DType; diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/AllowUnknownIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/AllowUnknownIntegrationTest.java index 4ef3aaec..35809f9a 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/AllowUnknownIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/AllowUnknownIntegrationTest.java @@ -5,7 +5,7 @@ import dev.vortex.arrow.ArrowAllocation; import dev.vortex.jni.NativeLoader; import io.github.dfa1.vortex.core.error.VortexException; -import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.Chunk; import io.github.dfa1.vortex.reader.array.UnknownArray; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.VortexReader; @@ -87,8 +87,8 @@ void allowUnknown_emptyRegistry_allColumnsReturnUnknownArray(@TempDir Path tmp) iter.forEachRemaining(c -> { totalRows.addAndGet(c.rowCount()); chunkCount.incrementAndGet(); - for (Array col : c.columns().values()) { - if (!(col instanceof UnknownArray)) { + for (Chunk.Column col : c.columns().values()) { + if (!(col.array() instanceof UnknownArray)) { allUnknown.set(false); } } @@ -130,8 +130,8 @@ void allowUnknown_loadAllRegistry_noUnknownArrayForSupportedEncodings(@TempDir P var iter = vf.scan(io.github.dfa1.vortex.reader.ScanOptions.all())) { iter.forEachRemaining(c -> { chunkCount.incrementAndGet(); - for (Array col : c.columns().values()) { - if (col instanceof UnknownArray) { + for (Chunk.Column col : c.columns().values()) { + if (col.array() instanceof UnknownArray) { anyUnknown.set(true); } } diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/JavaRoundTripIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/JavaRoundTripIntegrationTest.java index 9c253dd0..dcad84e0 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/JavaRoundTripIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/JavaRoundTripIntegrationTest.java @@ -65,7 +65,7 @@ private static int[] readIntColumn(Path file, String column) throws IOException var iter = vf.scan(ScanOptions.columns(column))) { var ints = new ArrayList(); iter.forEachRemaining(c -> { - IntArray arr = (IntArray) c.columns().get(column); + IntArray arr = c.column(column); for (long i = 0; i < arr.length(); i++) { ints.add(arr.getInt(i)); } diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/RustJavaReaderComparisonIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/RustJavaReaderComparisonIntegrationTest.java index 77249bc8..4ea8714f 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/RustJavaReaderComparisonIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/RustJavaReaderComparisonIntegrationTest.java @@ -21,6 +21,7 @@ import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.inspect.VortexInspector; import io.github.dfa1.vortex.reader.VortexReader; +import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.reader.Chunk; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.vector.BigIntVector; @@ -229,18 +230,19 @@ private static Stats javaStats(Path file) throws Exception { while (iter.hasNext()) { try (Chunk chunk = iter.next()) { rowCount += chunk.rowCount(); - for (Map.Entry e : chunk.columns().entrySet()) { - if (extensionCols.contains(e.getKey())) { + for (Map.Entry e : chunk.columns().entrySet()) { + String name = e.getKey().value(); + if (extensionCols.contains(name)) { continue; } - Array arr = e.getValue(); + Array arr = e.getValue().array(); Double numSum = numericSum(arr); if (numSum != null) { - numSums.merge(e.getKey(), numSum, Double::sum); + numSums.merge(name, numSum, Double::sum); } Long strLen = stringByteLength(arr); if (strLen != null && strLen > 0) { - strLenSums.merge(e.getKey(), strLen, Long::sum); + strLenSums.merge(name, strLen, Long::sum); } } } diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/RustWritesJavaReadsIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/RustWritesJavaReadsIntegrationTest.java index f2db6d77..a37d1c58 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/RustWritesJavaReadsIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/RustWritesJavaReadsIntegrationTest.java @@ -125,7 +125,7 @@ private static List scanAll(VortexReader vf, iter.forEachRemaining(c -> { var mat = new LinkedHashMap(c.columns().size()); for (var e : c.columns().entrySet()) { - mat.put(e.getKey(), snapshotArray(e.getValue())); + mat.put(e.getKey().value(), snapshotArray(e.getValue().array())); } results.add(new JavaChunk(c.rowCount(), mat)); }); @@ -212,7 +212,7 @@ private static long[] readJavaLongColumn(Path file, String column) throws IOExce var iter = vf.scan(io.github.dfa1.vortex.reader.ScanOptions.columns(column))) { var longs = new ArrayList(); iter.forEachRemaining(c -> { - LongArray arr = (LongArray) c.columns().get(column); + LongArray arr = c.column(column); for (long i = 0; i < arr.length(); i++) { longs.add(arr.getLong(i)); } diff --git a/performance/src/main/java/io/github/dfa1/vortex/performance/JniWritesJavaReadsBigFileBenchmark.java b/performance/src/main/java/io/github/dfa1/vortex/performance/JniWritesJavaReadsBigFileBenchmark.java index 478f5074..126bc9db 100644 --- a/performance/src/main/java/io/github/dfa1/vortex/performance/JniWritesJavaReadsBigFileBenchmark.java +++ b/performance/src/main/java/io/github/dfa1/vortex/performance/JniWritesJavaReadsBigFileBenchmark.java @@ -179,7 +179,7 @@ private long scanJava() throws IOException { var iter = vf.scan(io.github.dfa1.vortex.reader.ScanOptions.columns("c0"))) { while (iter.hasNext()) { try (Chunk c = iter.next()) { - Array arr = c.columns().get("c0"); + Array arr = c.column("c0"); MemorySegment buf = arr.materialize(Arena.ofAuto()); long count = buf.byteSize() / Long.BYTES; for (long i = 0; i < count; i++) { diff --git a/performance/src/main/java/io/github/dfa1/vortex/performance/TaxiColumnTreeDiff.java b/performance/src/main/java/io/github/dfa1/vortex/performance/TaxiColumnTreeDiff.java index 149a6496..cc29f8b6 100644 --- a/performance/src/main/java/io/github/dfa1/vortex/performance/TaxiColumnTreeDiff.java +++ b/performance/src/main/java/io/github/dfa1/vortex/performance/TaxiColumnTreeDiff.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.performance; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_INT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; import dev.vortex.api.Session; import dev.vortex.jni.NativeLoader; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/Chunk.java b/reader/src/main/java/io/github/dfa1/vortex/reader/Chunk.java index 96a224e2..52413b4e 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/Chunk.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/Chunk.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.reader; +import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.reader.array.Array; @@ -14,8 +15,8 @@ import java.time.LocalDate; import java.time.LocalTime; import java.util.List; -import java.util.Map; import java.util.Objects; +import java.util.SequencedMap; import java.util.UUID; import java.util.function.Consumer; @@ -26,6 +27,11 @@ /// references into that arena (or into the underlying mmap region), valid only /// while this `Chunk` is open. /// +/// Columns are keyed by [ColumnName] — the validated name domain the file parser +/// certifies at the read boundary, so every key here is policy-valid and unique. +/// [#column(String)] keeps a string-sugar overload for callers that hold a raw +/// name; it wraps the string in a [ColumnName] before looking it up. +/// /// **Lifecycle.** `Chunk` is [AutoCloseable]. Always wrap consumption in /// try-with-resources: /// @@ -43,21 +49,37 @@ public final class Chunk implements AutoCloseable { private final long rowCount; - private final Map columns; - private final Map columnDtypes; + private final SequencedMap columns; private final Arena arena; private final Consumer onClose; private boolean closed; - Chunk(long rowCount, Map columns, Map columnDtypes, + Chunk(long rowCount, SequencedMap columns, Arena arena, Consumer onClose) { this.rowCount = rowCount; this.columns = Objects.requireNonNull(columns); - this.columnDtypes = Objects.requireNonNull(columnDtypes); this.arena = Objects.requireNonNull(arena); this.onClose = Objects.requireNonNull(onClose); } + /// One decoded column: its zero-copy [Array] view and the [DType] it was declared with in the + /// file's schema. The dtype travels with the array so extension decoding (see [#as(String, Class)]) + /// and dtype-aware tooling need no second lookup. + /// + /// @param array the decoded column values, valid only while the owning [Chunk] is open + /// @param dtype the column's declared type from the file schema + public record Column(Array array, DType dtype) { + + /// Binds a decoded array to its declared type. + /// + /// @param array the decoded column values + /// @param dtype the column's declared type + public Column { + Objects.requireNonNull(array, "array"); + Objects.requireNonNull(dtype, "dtype"); + } + } + /// Number of logical rows in this chunk (after any limit truncation). /// /// @return row count of this chunk @@ -65,28 +87,42 @@ public long rowCount() { return rowCount; } - /// Returns the decoded columns by name. The map is unmodifiable; values are - /// valid only while this `Chunk` is open. + /// Returns the decoded columns in schema (projection) order. The map is unmodifiable and + /// preserves encounter order; each [Column] value is valid only while this `Chunk` is open. /// - /// @return projected columns keyed by name - public Map columns() { + /// @return projected columns keyed by [ColumnName], in schema order + public SequencedMap columns() { return columns; } - /// Looks up a column by name with a checked cast to the caller's expected - /// [Array] subtype. + /// Looks up a column by its raw string name with a checked cast to the caller's expected + /// [Array] subtype. The name is validated through [ColumnName#of(String)] first, so a + /// policy-invalid query name fails fast with the policy's [IllegalArgumentException] — it + /// could never match a certified column anyway. /// /// @param name column name as declared in the file's [io.github.dfa1.vortex.core.model.DType] schema /// @param expected concrete [Array] subtype /// @return the column array + /// @throws IllegalArgumentException if `name` violates the column-name policy + /// @throws VortexException if no column with the given name is present in this chunk + public T column(String name) { + return column(ColumnName.of(name)); + } + + /// Looks up a column by its validated [ColumnName] with a checked cast to the caller's + /// expected [Array] subtype. + /// + /// @param name validated column name as declared in the file's schema + /// @param expected concrete [Array] subtype + /// @return the column array /// @throws VortexException if no column with the given name is present in this chunk @SuppressWarnings("unchecked") - public T column(String name) { - Array arr = columns.get(name); - if (arr == null) { + public T column(ColumnName name) { + Column col = columns.get(name); + if (col == null) { throw new VortexException("unknown column: " + name); } - return (T) arr; + return (T) col.array(); } /// Decodes an extension column into a typed `List` of domain values. @@ -108,18 +144,20 @@ public T column(String name) { /// @return decoded values in row order /// @throws VortexException if `name` isn't present, isn't an extension column, /// or the requested `domainType` doesn't match the column's extension id + /// @throws IllegalArgumentException if `name` violates the column-name policy + /// ([io.github.dfa1.vortex.core.model.ColumnName]) — it could never match @SuppressWarnings("unchecked") public List as(String name, Class domainType) { - DType colDtype = columnDtypes.get(name); - if (colDtype == null) { + Column col = columns.get(ColumnName.of(name)); + if (col == null) { throw new VortexException("unknown column: " + name); } - if (!(colDtype instanceof DType.Extension ext)) { + if (!(col.dtype() instanceof DType.Extension ext)) { throw new VortexException("not an extension column: " + name); } ExtensionId id = ExtensionId.parse(ext.extensionId()) .orElseThrow(() -> new VortexException("not a spec extension id: " + ext.extensionId())); - Array storage = column(name); + Array storage = col.array(); Object result = switch (id) { case VORTEX_DATE -> { requireDomainType(name, domainType, LocalDate.class); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java index 73da1de1..cb20bce7 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java @@ -1,6 +1,7 @@ package io.github.dfa1.vortex.reader; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_INT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; +import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.io.IoBounds; import io.github.dfa1.vortex.core.error.VortexException; @@ -33,12 +34,14 @@ import java.lang.foreign.SegmentAllocator; import java.lang.foreign.ValueLayout; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.SequencedMap; import java.util.function.Consumer; /// Iterates over decoded chunks from a [io.github.dfa1.vortex.reader.VortexReader]. @@ -73,17 +76,17 @@ public final class ScanIterator implements Iterator, AutoCloseable { private final ScanOptions options; private List chunks; - private List projectedNames; + private List projectedNames; private List projectedDtypes; - private Map columnTopLayouts; - private Map columnDtypes; + private Map columnTopLayouts; + private Map columnDtypes; private int chunkIndex; private int peekedChunkIdx = -1; private long rowsReturned; private Chunk openChunk; private boolean closed; private Arena sharedArena; - private Map sharedFullArrays; + private Map sharedFullArrays; public ScanIterator(VortexHandle file, ScanOptions options) { this.file = file; @@ -112,11 +115,11 @@ private static void collectFlats(Layout layout, List out) { } } - private static List buildChunks(Map> columnFlats) { + private static List buildChunks(Map> columnFlats) { if (columnFlats.isEmpty()) { return List.of(); } - String[] colNames = columnFlats.keySet().toArray(String[]::new); + ColumnName[] colNames = columnFlats.keySet().toArray(ColumnName[]::new); int numCols = colNames.length; int maxChunks = 0; int refCol = 0; @@ -154,7 +157,7 @@ private static List buildChunks(Map> columnFlats return List.copyOf(result); } - private static ChunkSpec buildChunkSpec(String[] colNames, Map> columnFlats, + private static ChunkSpec buildChunkSpec(ColumnName[] colNames, Map> columnFlats, boolean[] shared, int chunkIdx, long sliceStart, long chunkRowCount) { int numCols = colNames.length; Layout[] layouts = new Layout[numCols]; @@ -176,37 +179,45 @@ private static ChunkSpec buildChunkSpec(String[] colNames, Map(); if (file.dtype() instanceof DType.Struct struct) { for (int i = 0; i < struct.fieldNames().size(); i++) { - columnDtypes.put(struct.fieldNames().get(i), struct.fieldTypes().get(i)); + columnDtypes.put(ColumnName.of(struct.fieldNames().get(i)), struct.fieldTypes().get(i)); } } } return columnDtypes.get(col); } - private static Map expandStruct(StructArray sa) { + private static SequencedMap expandStruct(StructArray sa) { DType.Struct sd = (DType.Struct) sa.dtype(); List names = sd.fieldNames(); + List types = sd.fieldTypes(); int n = names.size(); - var map = new LinkedHashMap(n); + var map = new LinkedHashMap(n); for (int i = 0; i < n; i++) { - map.put(names.get(i), sa.field(i)); + map.put(ColumnName.of(names.get(i)), new Chunk.Column(sa.field(i), types.get(i))); } - return Map.copyOf(map); + return unmodifiable(map); } // ── Zone-map pruning ────────────────────────────────────────────────────── - private static Map limitedColumns(Map columns, long rows) { - var result = new LinkedHashMap(columns.size()); + private static SequencedMap limitedColumns( + SequencedMap columns, long rows) { + var result = new LinkedHashMap(columns.size()); for (var entry : columns.entrySet()) { - result.put(entry.getKey(), Array.limited(entry.getValue(), rows)); + Chunk.Column col = entry.getValue(); + result.put(entry.getKey(), new Chunk.Column(Array.limited(col.array(), rows), col.dtype())); } - return Map.copyOf(result); + return unmodifiable(result); + } + + private static SequencedMap unmodifiable( + SequencedMap map) { + return Collections.unmodifiableSequencedMap(map); } @@ -254,12 +265,12 @@ public Chunk next() { Arena arena = Arena.ofConfined(); try { - Map columns = buildColumnMap(spec, arena); + SequencedMap columns = buildColumnMap(spec, arena); if (chunkRows < spec.rowCount()) { columns = limitedColumns(columns, chunkRows); } rowsReturned += chunkRows; - Chunk chunk = new Chunk(chunkRows, columns, projectedDtypeMap(), arena, this::onChunkClosed); + Chunk chunk = new Chunk(chunkRows, columns, arena, this::onChunkClosed); openChunk = chunk; return chunk; } catch (RuntimeException e) { @@ -303,22 +314,14 @@ Chunk decodeChunkAt(int chunkIndex) { ChunkSpec spec = chunks.get(chunkIndex); Arena arena = Arena.ofConfined(); try { - Map columns = buildSelfContainedColumnMap(spec, arena); - return new Chunk(spec.rowCount(), columns, projectedDtypeMap(), arena, c -> { }); + SequencedMap columns = buildSelfContainedColumnMap(spec, arena); + return new Chunk(spec.rowCount(), columns, arena, c -> { }); } catch (RuntimeException e) { arena.close(); throw e; } } - private Map projectedDtypeMap() { - Map map = new LinkedHashMap<>(projectedNames.size()); - for (int i = 0; i < projectedNames.size(); i++) { - map.put(projectedNames.get(i), projectedDtypes.get(i)); - } - return map; - } - /// Returns the row count of every chunk in scan order, without decoding values. /// /// Walks the file's layout tree (initializing internal state on first call) and @@ -371,14 +374,15 @@ public List columnZoneStats(String column) { if (chunks == null) { initialize(); } - List fromTable = decodeZoneTable(column); + ColumnName name = ColumnName.of(column); + List fromTable = decodeZoneTable(name); if (fromTable != null) { return fromTable; } // No zone-map table — surface each chunk's embedded ArrayStats (sum absent). List out = new ArrayList<>(chunks.size()); for (ChunkSpec spec : chunks) { - Layout flat = spec.layoutFor(column); + Layout flat = spec.layoutFor(name); out.add(flat == null ? ArrayStats.empty() : readFlatStats(flat)); } return out; @@ -389,7 +393,7 @@ public List columnZoneStats(String column) { /// node stats). The table is a single flat segment encoding a struct with a subset of the /// `min`/`max`/`sum`/`null_count` fields (see [ZonedStatsSchema]); it is decoded into a /// short-lived confined arena and the scalar values are boxed out before the arena closes. - private List decodeZoneTable(String column) { + private List decodeZoneTable(ColumnName column) { Layout zoned = findZonedLayout(file.layout(), column); if (zoned == null || zoned.children().size() < 2) { return null; @@ -435,11 +439,11 @@ private List decodeZoneTable(String column) { /// Finds the first `vortex.stats` layout in the subtree of `column`'s top-level layout, or /// `null` when the column is not zone-mapped. - private Layout findZonedLayout(Layout root, String column) { + private Layout findZonedLayout(Layout root, ColumnName column) { if (!(file.dtype() instanceof DType.Struct struct) || !root.isStruct()) { return null; } - int idx = struct.fieldNames().indexOf(column); + int idx = struct.fieldNames().indexOf(column.value()); if (idx < 0 || idx >= root.children().size()) { return null; } @@ -533,18 +537,21 @@ private void initialize() { Layout rootLayout = file.layout(); DType rootDtype = file.dtype(); - var columnFlats = new LinkedHashMap>(); - var columnTopLayouts = new LinkedHashMap(); - Map columnDtypes = new LinkedHashMap<>(); + var columnFlats = new LinkedHashMap>(); + var columnTopLayouts = new LinkedHashMap(); + Map columnDtypes = new LinkedHashMap<>(); if (rootLayout.isStruct() && rootDtype instanceof DType.Struct structDtype) { List projection = options.columns(); for (int i = 0; i < rootLayout.children().size(); i++) { - String colName = structDtype.fieldNames().get(i); + String rawName = structDtype.fieldNames().get(i); DType colDtype = structDtype.fieldTypes().get(i); - if (!projection.isEmpty() && !projection.contains(colName)) { + if (!projection.isEmpty() && !projection.contains(rawName)) { continue; } + // File-schema names are already policy-certified by PostscriptParser, so this + // ColumnName construction never throws — it just types the certified key. + ColumnName colName = ColumnName.of(rawName); Layout colTop = rootLayout.children().get(i); var flats = new ArrayList(); collectFlats(colTop, flats); @@ -555,9 +562,10 @@ private void initialize() { } else { var flats = new ArrayList(); collectFlats(rootLayout, flats); - columnFlats.put("_col", flats); - columnTopLayouts.put("_col", rootLayout); - columnDtypes.put("_col", rootDtype); + ColumnName colName = ColumnName.of("_col"); + columnFlats.put(colName, flats); + columnTopLayouts.put(colName, rootLayout); + columnDtypes.put(colName, rootDtype); } projectedNames = List.copyOf(columnDtypes.keySet()); @@ -568,9 +576,9 @@ private void initialize() { } private void decodeSharedColumns( - Map> columnFlats, - Map columnTopLayouts, - Map columnDtypes) { + Map> columnFlats, + Map columnTopLayouts, + Map columnDtypes) { int maxFlats = 0; for (List flats : columnFlats.values()) { if (flats.size() > maxFlats) { @@ -586,19 +594,18 @@ private void decodeSharedColumns( } if (sharedArena == null) { sharedArena = Arena.ofConfined(); - sharedFullArrays = new java.util.HashMap<>(); + sharedFullArrays = new HashMap<>(); } - String name = entry.getKey(); + ColumnName name = entry.getKey(); Layout topLayout = columnTopLayouts.get(name); DType dtype = columnDtypes.get(name); sharedFullArrays.put(name, decodeLayout(topLayout, dtype, sharedArena)); } } - // Map.of with 1 or 2 args allocates Map1/Map2 (~2-4 fields) — avoids the - // LinkedHashMap + Map.copyOf pair that would otherwise allocate per chunk. - // Direct array index into ChunkSpec.columnLayouts avoids HashMap.get() per column. - private Map buildColumnMap(ChunkSpec chunk, Arena arena) { + // A LinkedHashMap preserves schema/projection order (the public columns() contract is a + // SequencedMap). Direct array index into ChunkSpec.columnLayouts avoids HashMap.get() per column. + private SequencedMap buildColumnMap(ChunkSpec chunk, Arena arena) { Layout[] layouts = chunk.columnLayouts(); long[] sliceOffsets = chunk.sliceOffsets(); int n = projectedNames.size(); @@ -607,27 +614,21 @@ private Map buildColumnMap(ChunkSpec chunk, Arena arena) { if (arr instanceof StructArray sa) { return expandStruct(sa); } - return Map.of(projectedNames.getFirst(), arr); - } - if (n == 2) { - return Map.of( - projectedNames.get(0), - decodeOrSlice(0, layouts[0], sliceOffsets[0], chunk.rowCount(), arena), - projectedNames.get(1), - decodeOrSlice(1, layouts[1], sliceOffsets[1], chunk.rowCount(), arena)); + return singleColumn(arr); } - var scratch = new LinkedHashMap(n); + var scratch = new LinkedHashMap(n); for (int i = 0; i < n; i++) { - scratch.put(projectedNames.get(i), - decodeOrSlice(i, layouts[i], sliceOffsets[i], chunk.rowCount(), arena)); + scratch.put(projectedNames.get(i), new Chunk.Column( + decodeOrSlice(i, layouts[i], sliceOffsets[i], chunk.rowCount(), arena), + projectedDtypes.get(i))); } - return Map.copyOf(scratch); + return unmodifiable(scratch); } /// Builds the column map for [#decodeChunkAt(int)]. Identical decode to [#buildColumnMap] /// except that a shared (single-flat) column is decoded into `arena` and sliced there, /// so the resulting [Chunk] owns every buffer and survives this iterator's close. - private Map buildSelfContainedColumnMap(ChunkSpec chunk, Arena arena) { + private SequencedMap buildSelfContainedColumnMap(ChunkSpec chunk, Arena arena) { Layout[] layouts = chunk.columnLayouts(); long[] sliceOffsets = chunk.sliceOffsets(); int n = projectedNames.size(); @@ -636,14 +637,21 @@ private Map buildSelfContainedColumnMap(ChunkSpec chunk, Arena ar if (arr instanceof StructArray sa) { return expandStruct(sa); } - return Map.of(projectedNames.getFirst(), arr); + return singleColumn(arr); } - var scratch = new LinkedHashMap(n); + var scratch = new LinkedHashMap(n); for (int i = 0; i < n; i++) { - scratch.put(projectedNames.get(i), - decodeOrSliceSelfContained(i, layouts[i], sliceOffsets[i], chunk.rowCount(), arena)); + scratch.put(projectedNames.get(i), new Chunk.Column( + decodeOrSliceSelfContained(i, layouts[i], sliceOffsets[i], chunk.rowCount(), arena), + projectedDtypes.get(i))); } - return Map.copyOf(scratch); + return unmodifiable(scratch); + } + + private SequencedMap singleColumn(Array array) { + var map = new LinkedHashMap(1); + map.put(projectedNames.getFirst(), new Chunk.Column(array, projectedDtypes.getFirst())); + return unmodifiable(map); } private Array decodeOrSliceSelfContained(int colIdx, Layout layout, long sliceStart, @@ -653,7 +661,7 @@ private Array decodeOrSliceSelfContained(int colIdx, Layout layout, long sliceSt } // Shared single-flat column: decode its full top layout into THIS chunk's arena and // slice, so the returned Chunk does not reference the iterator's shared arena. - String name = projectedNames.get(colIdx); + ColumnName name = projectedNames.get(colIdx); DType dtype = projectedDtypes.get(colIdx); Array full = decodeLayout(columnTopLayouts.get(name), dtype, arena); return sliceArray(full, sliceStart, rowCount, dtype); @@ -713,12 +721,15 @@ private boolean canPruneChunk(ChunkSpec chunk, RowFilter filter) { yield false; } case RowFilter.Column(var col, var predicate) -> { - Layout flat = chunk.layoutFor(col); + // The filter's raw column string enters the typed internals here; a valid-but-absent + // name yields no layout (no pruning), matching the row-level scan it gates. + ColumnName name = ColumnName.of(col); + Layout flat = chunk.layoutFor(name); if (flat == null) { yield false; } ArrayStats stats = readFlatStats(flat); - yield canPrune(predicate, stats, flat.rowCount(), columnDType(col)); + yield canPrune(predicate, stats, flat.rowCount(), columnDType(name)); } }; } @@ -838,8 +849,8 @@ public SegmentSpec segmentSpec(int index) { @SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. private record ChunkSpec( - long rowCount, String[] columnNames, Layout[] columnLayouts, long[] sliceOffsets) { - Layout layoutFor(String col) { + long rowCount, ColumnName[] columnNames, Layout[] columnLayouts, long[] sliceOffsets) { + Layout layoutFor(ColumnName col) { for (int i = 0; i < columnNames.length; i++) { if (columnNames[i].equals(col)) { return columnLayouts[i]; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/SerializedArrayDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/SerializedArrayDecoder.java index c6a0003b..2bc9b236 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/SerializedArrayDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/SerializedArrayDecoder.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.reader; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_INT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.DType; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/Trailer.java b/reader/src/main/java/io/github/dfa1/vortex/reader/Trailer.java index 8d95658d..38b5f250 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/Trailer.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/Trailer.java @@ -6,7 +6,8 @@ import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.ByteOrder; + +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_SHORT; /// Parsed Vortex file trailer. /// @@ -18,8 +19,6 @@ /// @param postscriptLen length in bytes of the postscript blob immediately preceding the trailer record Trailer(int version, int postscriptLen) { - private static final ValueLayout.OfShort LE_SHORT = - ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); /// Parse the 8-byte trailer from a [MemorySegment] view and validate magic, version, and /// postscript length against the body size. diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java b/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java index d0fd0ff5..d4dae205 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.reader; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_INT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.io.IoBounds; import io.github.dfa1.vortex.core.error.VortexException; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictDoubleArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictDoubleArray.java index 24db578d..a4ffb26a 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictDoubleArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictDoubleArray.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.error.VortexException; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -63,22 +63,22 @@ public MemorySegment materialize(SegmentAllocator arena) { switch (codes) { case ByteArray ba -> { for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_DOUBLE, i, vals.getDouble(Byte.toUnsignedLong(ba.getByte(i)))); + dst.setAtIndex(VortexFormat.LE_DOUBLE, i, vals.getDouble(Byte.toUnsignedLong(ba.getByte(i)))); } } case ShortArray sa -> { for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_DOUBLE, i, vals.getDouble(Short.toUnsignedLong(sa.getShort(i)))); + dst.setAtIndex(VortexFormat.LE_DOUBLE, i, vals.getDouble(Short.toUnsignedLong(sa.getShort(i)))); } } case IntArray ia -> { for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_DOUBLE, i, vals.getDouble(Integer.toUnsignedLong(ia.getInt(i)))); + dst.setAtIndex(VortexFormat.LE_DOUBLE, i, vals.getDouble(Integer.toUnsignedLong(ia.getInt(i)))); } } case LongArray la -> { for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_DOUBLE, i, vals.getDouble(la.getLong(i))); + dst.setAtIndex(VortexFormat.LE_DOUBLE, i, vals.getDouble(la.getLong(i))); } } default -> throw new VortexException("DictDoubleArray: invalid codes type: " diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictFloatArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictFloatArray.java index a675c100..50c245ab 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictFloatArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictFloatArray.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.error.VortexException; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -62,22 +62,22 @@ public MemorySegment materialize(SegmentAllocator arena) { switch (codes) { case ByteArray ba -> { for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_FLOAT, i, vals.getFloat(Byte.toUnsignedLong(ba.getByte(i)))); + dst.setAtIndex(VortexFormat.LE_FLOAT, i, vals.getFloat(Byte.toUnsignedLong(ba.getByte(i)))); } } case ShortArray sa -> { for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_FLOAT, i, vals.getFloat(Short.toUnsignedLong(sa.getShort(i)))); + dst.setAtIndex(VortexFormat.LE_FLOAT, i, vals.getFloat(Short.toUnsignedLong(sa.getShort(i)))); } } case IntArray ia -> { for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_FLOAT, i, vals.getFloat(Integer.toUnsignedLong(ia.getInt(i)))); + dst.setAtIndex(VortexFormat.LE_FLOAT, i, vals.getFloat(Integer.toUnsignedLong(ia.getInt(i)))); } } case LongArray la -> { for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_FLOAT, i, vals.getFloat(la.getLong(i))); + dst.setAtIndex(VortexFormat.LE_FLOAT, i, vals.getFloat(la.getLong(i))); } } default -> throw new VortexException("DictFloatArray: invalid codes type: " diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictIntArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictIntArray.java index 472f7a11..870bd159 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictIntArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictIntArray.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.error.VortexException; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -63,22 +63,22 @@ public MemorySegment materialize(SegmentAllocator arena) { switch (codes) { case ByteArray ba -> { for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_INT, i, vals.getInt(Byte.toUnsignedLong(ba.getByte(i)))); + dst.setAtIndex(VortexFormat.LE_INT, i, vals.getInt(Byte.toUnsignedLong(ba.getByte(i)))); } } case ShortArray sa -> { for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_INT, i, vals.getInt(Short.toUnsignedLong(sa.getShort(i)))); + dst.setAtIndex(VortexFormat.LE_INT, i, vals.getInt(Short.toUnsignedLong(sa.getShort(i)))); } } case IntArray ia -> { for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_INT, i, vals.getInt(Integer.toUnsignedLong(ia.getInt(i)))); + dst.setAtIndex(VortexFormat.LE_INT, i, vals.getInt(Integer.toUnsignedLong(ia.getInt(i)))); } } case LongArray la -> { for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_INT, i, vals.getInt(la.getLong(i))); + dst.setAtIndex(VortexFormat.LE_INT, i, vals.getInt(la.getLong(i))); } } default -> throw new VortexException("DictIntArray: invalid codes type: " diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictLongArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictLongArray.java index 40deab7d..33868c43 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictLongArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DictLongArray.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.error.VortexException; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -66,22 +66,22 @@ public MemorySegment materialize(SegmentAllocator arena) { switch (codes) { case ByteArray ba -> { for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_LONG, i, vals.getLong(Byte.toUnsignedLong(ba.getByte(i)))); + dst.setAtIndex(VortexFormat.LE_LONG, i, vals.getLong(Byte.toUnsignedLong(ba.getByte(i)))); } } case ShortArray sa -> { for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_LONG, i, vals.getLong(Short.toUnsignedLong(sa.getShort(i)))); + dst.setAtIndex(VortexFormat.LE_LONG, i, vals.getLong(Short.toUnsignedLong(sa.getShort(i)))); } } case IntArray ia -> { for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_LONG, i, vals.getLong(Integer.toUnsignedLong(ia.getInt(i)))); + dst.setAtIndex(VortexFormat.LE_LONG, i, vals.getLong(Integer.toUnsignedLong(ia.getInt(i)))); } } case LongArray la -> { for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_LONG, i, vals.getLong(la.getLong(i))); + dst.setAtIndex(VortexFormat.LE_LONG, i, vals.getLong(la.getLong(i))); } } default -> throw new VortexException("DictLongArray: invalid codes type: " diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/DoubleArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DoubleArray.java index 2291ae90..6f4672b1 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/DoubleArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/DoubleArray.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.reader.array; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -67,7 +67,7 @@ default MemorySegment materialize(SegmentAllocator arena) { long n = length(); MemorySegment dst = arena.allocate(n * 8L, 8); for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_DOUBLE, i, getDouble(i)); + dst.setAtIndex(VortexFormat.LE_DOUBLE, i, getDouble(i)); } return dst; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/FloatArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/FloatArray.java index 1b19e96e..448dfb5c 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/FloatArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/FloatArray.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.reader.array; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -56,7 +56,7 @@ default MemorySegment materialize(SegmentAllocator arena) { long n = length(); MemorySegment dst = arena.allocate(n * 4L, 4); for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_FLOAT, i, getFloat(i)); + dst.setAtIndex(VortexFormat.LE_FLOAT, i, getFloat(i)); } return dst; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/GenericArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/GenericArray.java index 2c22ff8f..49ab5c87 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/GenericArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/GenericArray.java @@ -8,10 +8,13 @@ import java.lang.foreign.ValueLayout; import java.math.BigDecimal; import java.math.BigInteger; -import java.nio.ByteOrder; import java.util.Objects; import java.util.Optional; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_LONG; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_SHORT; + /// Fallback [Array] for dtypes that lack a dedicated concrete subtype. /// /// Holds raw buffer segments and child arrays. Used by encodings during migration @@ -135,19 +138,13 @@ private static BigInteger readSingleBufferMantissa(MemorySegment buf, long lengt return readSignedLe(buf, i * width, width); } - private static final ValueLayout.OfShort SHORT_LE = - ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); - private static final ValueLayout.OfInt INT_LE = - ValueLayout.JAVA_INT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); - private static final ValueLayout.OfLong LONG_LE = - ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); private static BigInteger readSignedLe(MemorySegment buf, long offset, int width) { return switch (width) { case 1 -> BigInteger.valueOf(buf.get(ValueLayout.JAVA_BYTE, offset)); - case 2 -> BigInteger.valueOf(buf.get(SHORT_LE, offset)); - case 4 -> BigInteger.valueOf(buf.get(INT_LE, offset)); - case 8 -> BigInteger.valueOf(buf.get(LONG_LE, offset)); + case 2 -> BigInteger.valueOf(buf.get(LE_SHORT, offset)); + case 4 -> BigInteger.valueOf(buf.get(LE_INT, offset)); + case 8 -> BigInteger.valueOf(buf.get(LE_LONG, offset)); case 16 -> readSigned128Le(buf, offset); default -> throw new VortexException("readSignedLe: unsupported width " + width); }; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/IntArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/IntArray.java index 9561bf58..74cd3c9a 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/IntArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/IntArray.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.reader.array; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -67,7 +67,7 @@ default MemorySegment materialize(SegmentAllocator arena) { long n = length(); MemorySegment dst = arena.allocate(n * 4L, 4); for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_INT, i, getInt(i)); + dst.setAtIndex(VortexFormat.LE_INT, i, getInt(i)); } return dst; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyAlpDoubleArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyAlpDoubleArray.java index 910eaa67..2785b4a2 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyAlpDoubleArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyAlpDoubleArray.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.reader.array; import io.github.dfa1.vortex.core.model.DType; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -28,7 +28,7 @@ public record LazyAlpDoubleArray(DType dtype, long length, MemorySegment encoded @Override public double getDouble(long i) { - return encoded.getAtIndex(PTypeIO.LE_LONG, i) * factorF * factorE; + return encoded.getAtIndex(VortexFormat.LE_LONG, i) * factorF * factorE; } /// Bulk-decodes through [#getDouble(long)] into a fresh little-endian `f64` segment. @@ -44,7 +44,7 @@ public MemorySegment materialize(SegmentAllocator arena) { long n = length; MemorySegment dst = arena.allocate(n * 8L, 8); for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_DOUBLE, i, getDouble(i)); + dst.setAtIndex(VortexFormat.LE_DOUBLE, i, getDouble(i)); } return dst; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyAlpFloatArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyAlpFloatArray.java index 3e01eef5..870507cb 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyAlpFloatArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyAlpFloatArray.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.reader.array; import io.github.dfa1.vortex.core.model.DType; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -23,7 +23,7 @@ public record LazyAlpFloatArray(DType dtype, long length, MemorySegment encoded, @Override public float getFloat(long i) { - return encoded.getAtIndex(PTypeIO.LE_INT, i) * factorF * factorE; + return encoded.getAtIndex(VortexFormat.LE_INT, i) * factorF * factorE; } /// Bulk-decodes through [#getFloat(long)] into a fresh little-endian `f32` segment. @@ -39,7 +39,7 @@ public MemorySegment materialize(SegmentAllocator arena) { long n = length; MemorySegment dst = arena.allocate(n * 4L, 4); for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_FLOAT, i, getFloat(i)); + dst.setAtIndex(VortexFormat.LE_FLOAT, i, getFloat(i)); } return dst; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyConstantDecimalArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyConstantDecimalArray.java index f1ffadf8..e7971455 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyConstantDecimalArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyConstantDecimalArray.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.error.VortexException; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -53,9 +53,9 @@ public MemorySegment materialize(SegmentAllocator arena) { long off = i * byteWidth; switch (byteWidth) { case 1 -> dst.set(ValueLayout.JAVA_BYTE, off, (byte) rawBits); - case 2 -> dst.set(PTypeIO.LE_SHORT, off, (short) rawBits); - case 4 -> dst.set(PTypeIO.LE_INT, off, (int) rawBits); - case 8 -> dst.set(PTypeIO.LE_LONG, off, rawBits); + case 2 -> dst.set(VortexFormat.LE_SHORT, off, (short) rawBits); + case 4 -> dst.set(VortexFormat.LE_INT, off, (int) rawBits); + case 8 -> dst.set(VortexFormat.LE_LONG, off, rawBits); default -> throw new VortexException("LazyConstantDecimalArray: unsupported byteWidth " + byteWidth); } } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyDecimalArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyDecimalArray.java index 832a050a..46702796 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyDecimalArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyDecimalArray.java @@ -8,10 +8,13 @@ import java.lang.foreign.ValueLayout; import java.math.BigDecimal; import java.math.BigInteger; -import java.nio.ByteOrder; import java.util.Objects; import java.util.Optional; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_LONG; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_SHORT; + /// Lazy `vortex.decimal` array. /// /// `vortex.decimal` stores one little-endian two's-complement integer per row; @@ -27,12 +30,6 @@ /// @param byteWidth element width in bytes; one of 1, 2, 4, 8, 16 public record LazyDecimalArray(DType dtype, long length, MemorySegment buf, int byteWidth) implements DecimalArray { - private static final ValueLayout.OfShort SHORT_LE = - ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); - private static final ValueLayout.OfInt INT_LE = - ValueLayout.JAVA_INT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); - private static final ValueLayout.OfLong LONG_LE = - ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); /// Reads cell `i` as a [BigDecimal] with the parent dtype's scale. /// @@ -51,9 +48,9 @@ public BigDecimal getDecimal(long i) { private static BigInteger readSignedLe(MemorySegment buf, long offset, int width) { return switch (width) { case 1 -> BigInteger.valueOf(buf.get(ValueLayout.JAVA_BYTE, offset)); - case 2 -> BigInteger.valueOf(buf.get(SHORT_LE, offset)); - case 4 -> BigInteger.valueOf(buf.get(INT_LE, offset)); - case 8 -> BigInteger.valueOf(buf.get(LONG_LE, offset)); + case 2 -> BigInteger.valueOf(buf.get(LE_SHORT, offset)); + case 4 -> BigInteger.valueOf(buf.get(LE_INT, offset)); + case 8 -> BigInteger.valueOf(buf.get(LE_LONG, offset)); case 16 -> readSigned128Le(buf, offset); default -> throw new VortexException("LazyDecimalArray: unsupported element width " + width); }; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyForIntArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyForIntArray.java index d1bff410..6d99fd32 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyForIntArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyForIntArray.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.reader.array; import io.github.dfa1.vortex.core.model.DType; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -21,7 +21,7 @@ public record LazyForIntArray(DType dtype, long length, MemorySegment encoded, i @Override public int getInt(long i) { - return encoded.getAtIndex(PTypeIO.LE_INT, i) + ref; + return encoded.getAtIndex(VortexFormat.LE_INT, i) + ref; } /// Bulk-decodes through [#getInt(long)] into a fresh little-endian `i32` segment. @@ -36,7 +36,7 @@ public MemorySegment materialize(SegmentAllocator arena) { long n = length; MemorySegment dst = arena.allocate(n * 4L, 4); for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_INT, i, getInt(i)); + dst.setAtIndex(VortexFormat.LE_INT, i, getInt(i)); } return dst; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyForLongArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyForLongArray.java index f8b53eab..c89a1069 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyForLongArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyForLongArray.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.reader.array; import io.github.dfa1.vortex.core.model.DType; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -21,7 +21,7 @@ public record LazyForLongArray(DType dtype, long length, MemorySegment encoded, @Override public long getLong(long i) { - return encoded.getAtIndex(PTypeIO.LE_LONG, i) + ref; + return encoded.getAtIndex(VortexFormat.LE_LONG, i) + ref; } /// Bulk-decodes through [#getLong(long)] into a fresh little-endian `i64` segment. @@ -36,7 +36,7 @@ public MemorySegment materialize(SegmentAllocator arena) { long n = length; MemorySegment dst = arena.allocate(n * 8L, 8); for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_LONG, i, getLong(i)); + dst.setAtIndex(VortexFormat.LE_LONG, i, getLong(i)); } return dst; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyForShortArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyForShortArray.java index 4f5b41e7..44ec6b91 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyForShortArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyForShortArray.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.util.function.LongBinaryOperator; @@ -22,7 +22,7 @@ public record LazyForShortArray(DType dtype, long length, MemorySegment encoded, @Override public short getShort(long i) { - return (short) (encoded.getAtIndex(PTypeIO.LE_SHORT, i) + ref); + return (short) (encoded.getAtIndex(VortexFormat.LE_SHORT, i) + ref); } @Override @@ -33,7 +33,7 @@ public long fold(long identity, LongBinaryOperator op) { short r = ref; boolean unsigned = dtype instanceof DType.Primitive p && p.ptype() == PType.U16; for (long i = 0; i < n; i++) { - short v = (short) (seg.getAtIndex(PTypeIO.LE_SHORT, i) + r); + short v = (short) (seg.getAtIndex(VortexFormat.LE_SHORT, i) + r); result = op.applyAsLong(result, unsigned ? Short.toUnsignedLong(v) : (long) v); } return result; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyZigZagIntArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyZigZagIntArray.java index e1d5cb07..1a0818e3 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyZigZagIntArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyZigZagIntArray.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.reader.array; import io.github.dfa1.vortex.core.model.DType; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -20,7 +20,7 @@ public record LazyZigZagIntArray(DType dtype, long length, MemorySegment encoded @Override public int getInt(long i) { - int u = encoded.getAtIndex(PTypeIO.LE_INT, i); + int u = encoded.getAtIndex(VortexFormat.LE_INT, i); return (u >>> 1) ^ -(u & 1); } @@ -36,7 +36,7 @@ public MemorySegment materialize(SegmentAllocator arena) { long n = length; MemorySegment dst = arena.allocate(n * 4L, 4); for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_INT, i, getInt(i)); + dst.setAtIndex(VortexFormat.LE_INT, i, getInt(i)); } return dst; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyZigZagLongArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyZigZagLongArray.java index d6bae36a..ed1cce07 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyZigZagLongArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyZigZagLongArray.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.reader.array; import io.github.dfa1.vortex.core.model.DType; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -20,7 +20,7 @@ public record LazyZigZagLongArray(DType dtype, long length, MemorySegment encode @Override public long getLong(long i) { - long u = encoded.getAtIndex(PTypeIO.LE_LONG, i); + long u = encoded.getAtIndex(VortexFormat.LE_LONG, i); return (u >>> 1) ^ -(u & 1L); } @@ -36,7 +36,7 @@ public MemorySegment materialize(SegmentAllocator arena) { long n = length; MemorySegment dst = arena.allocate(n * 8L, 8); for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_LONG, i, getLong(i)); + dst.setAtIndex(VortexFormat.LE_LONG, i, getLong(i)); } return dst; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyZigZagShortArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyZigZagShortArray.java index d76fb919..07421da7 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyZigZagShortArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyZigZagShortArray.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.util.function.LongBinaryOperator; @@ -22,7 +22,7 @@ public record LazyZigZagShortArray(DType dtype, long length, MemorySegment encod @Override public short getShort(long i) { - int u = Short.toUnsignedInt(encoded.getAtIndex(PTypeIO.LE_SHORT, i)); + int u = Short.toUnsignedInt(encoded.getAtIndex(VortexFormat.LE_SHORT, i)); return (short) ((u >>> 1) ^ -(u & 1)); } @@ -33,7 +33,7 @@ public long fold(long identity, LongBinaryOperator op) { long result = identity; boolean unsigned = dtype instanceof DType.Primitive p && p.ptype() == PType.U16; for (long i = 0; i < n; i++) { - int u = Short.toUnsignedInt(seg.getAtIndex(PTypeIO.LE_SHORT, i)); + int u = Short.toUnsignedInt(seg.getAtIndex(VortexFormat.LE_SHORT, i)); short v = (short) ((u >>> 1) ^ -(u & 1)); result = op.applyAsLong(result, unsigned ? Short.toUnsignedLong(v) : (long) v); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LongArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LongArray.java index dc197ea7..2d29dd58 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/LongArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/LongArray.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.reader.array; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -67,7 +67,7 @@ default MemorySegment materialize(SegmentAllocator arena) { long n = length(); MemorySegment dst = arena.allocate(n * 8L, 8); for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_LONG, i, getLong(i)); + dst.setAtIndex(VortexFormat.LE_LONG, i, getLong(i)); } return dst; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedDoubleArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedDoubleArray.java index e404a5d1..f290d457 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedDoubleArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedDoubleArray.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.reader.array; import io.github.dfa1.vortex.core.model.DType; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.util.function.DoubleBinaryOperator; @@ -22,7 +22,7 @@ public MaterializedDoubleArray(DType dtype, long length, MemorySegment buffer) { @Override public double getDouble(long i) { - return buffer.getAtIndex(PTypeIO.LE_DOUBLE, length == elementCount ? i : i % elementCount); + return buffer.getAtIndex(VortexFormat.LE_DOUBLE, length == elementCount ? i : i % elementCount); } @Override @@ -31,11 +31,11 @@ public void forEachDouble(DoubleConsumer c) { long n = length; if (n == elementCount) { for (long i = 0; i < n; i++) { - c.accept(buf.getAtIndex(PTypeIO.LE_DOUBLE, i)); + c.accept(buf.getAtIndex(VortexFormat.LE_DOUBLE, i)); } } else { for (long i = 0; i < n; i++) { - c.accept(buf.getAtIndex(PTypeIO.LE_DOUBLE, i % elementCount)); + c.accept(buf.getAtIndex(VortexFormat.LE_DOUBLE, i % elementCount)); } } } @@ -47,11 +47,11 @@ public double fold(double identity, DoubleBinaryOperator op) { double result = identity; if (n == elementCount) { for (long i = 0; i < n; i++) { - result = op.applyAsDouble(result, buf.getAtIndex(PTypeIO.LE_DOUBLE, i)); + result = op.applyAsDouble(result, buf.getAtIndex(VortexFormat.LE_DOUBLE, i)); } } else { for (long i = 0; i < n; i++) { - result = op.applyAsDouble(result, buf.getAtIndex(PTypeIO.LE_DOUBLE, i % elementCount)); + result = op.applyAsDouble(result, buf.getAtIndex(VortexFormat.LE_DOUBLE, i % elementCount)); } } return result; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedFloat16Array.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedFloat16Array.java index d2f11bb9..c6cc733d 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedFloat16Array.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedFloat16Array.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.core.model.DType; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; @@ -21,7 +21,7 @@ public MaterializedFloat16Array(DType dtype, long length, MemorySegment buffer) @Override public float getFloat(long i) { - return Float.float16ToFloat(buffer.getAtIndex(PTypeIO.LE_SHORT, i)); + return Float.float16ToFloat(buffer.getAtIndex(VortexFormat.LE_SHORT, i)); } @Override diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedFloatArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedFloatArray.java index 436fc0d4..2f361c85 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedFloatArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedFloatArray.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.core.model.DType; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.util.function.DoubleBinaryOperator; @@ -22,7 +22,7 @@ public MaterializedFloatArray(DType dtype, long length, MemorySegment buffer) { @Override public float getFloat(long i) { - return buffer.getAtIndex(PTypeIO.LE_FLOAT, length == elementCount ? i : i % elementCount); + return buffer.getAtIndex(VortexFormat.LE_FLOAT, length == elementCount ? i : i % elementCount); } @Override @@ -32,11 +32,11 @@ public double fold(double identity, DoubleBinaryOperator op) { double result = identity; if (n == elementCount) { for (long i = 0; i < n; i++) { - result = op.applyAsDouble(result, buf.getAtIndex(PTypeIO.LE_FLOAT, i)); + result = op.applyAsDouble(result, buf.getAtIndex(VortexFormat.LE_FLOAT, i)); } } else { for (long i = 0; i < n; i++) { - result = op.applyAsDouble(result, buf.getAtIndex(PTypeIO.LE_FLOAT, i % elementCount)); + result = op.applyAsDouble(result, buf.getAtIndex(VortexFormat.LE_FLOAT, i % elementCount)); } } return result; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedIntArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedIntArray.java index 07219bb8..059ed738 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedIntArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedIntArray.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.core.model.DType; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.util.function.IntBinaryOperator; @@ -23,7 +23,7 @@ public MaterializedIntArray(DType dtype, long length, MemorySegment buffer) { @Override public int getInt(long i) { - return buffer.getAtIndex(PTypeIO.LE_INT, length == elementCount ? i : i % elementCount); + return buffer.getAtIndex(VortexFormat.LE_INT, length == elementCount ? i : i % elementCount); } @Override @@ -32,11 +32,11 @@ public void forEachInt(IntConsumer c) { long n = length; if (n == elementCount) { for (long i = 0; i < n; i++) { - c.accept(buf.getAtIndex(PTypeIO.LE_INT, i)); + c.accept(buf.getAtIndex(VortexFormat.LE_INT, i)); } } else { for (long i = 0; i < n; i++) { - c.accept(buf.getAtIndex(PTypeIO.LE_INT, i % elementCount)); + c.accept(buf.getAtIndex(VortexFormat.LE_INT, i % elementCount)); } } } @@ -48,11 +48,11 @@ public int fold(int identity, IntBinaryOperator op) { int result = identity; if (n == elementCount) { for (long i = 0; i < n; i++) { - result = op.applyAsInt(result, buf.getAtIndex(PTypeIO.LE_INT, i)); + result = op.applyAsInt(result, buf.getAtIndex(VortexFormat.LE_INT, i)); } } else { for (long i = 0; i < n; i++) { - result = op.applyAsInt(result, buf.getAtIndex(PTypeIO.LE_INT, i % elementCount)); + result = op.applyAsInt(result, buf.getAtIndex(VortexFormat.LE_INT, i % elementCount)); } } return result; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedLongArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedLongArray.java index c79820ca..a0013229 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedLongArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedLongArray.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.core.model.DType; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.util.function.LongBinaryOperator; @@ -23,7 +23,7 @@ public MaterializedLongArray(DType dtype, long length, MemorySegment buffer) { @Override public long getLong(long i) { - return buffer.getAtIndex(PTypeIO.LE_LONG, length == elementCount ? i : i % elementCount); + return buffer.getAtIndex(VortexFormat.LE_LONG, length == elementCount ? i : i % elementCount); } @Override @@ -32,11 +32,11 @@ public void forEachLong(LongConsumer c) { long n = length; if (n == elementCount) { for (long i = 0; i < n; i++) { - c.accept(buf.getAtIndex(PTypeIO.LE_LONG, i)); + c.accept(buf.getAtIndex(VortexFormat.LE_LONG, i)); } } else { for (long i = 0; i < n; i++) { - c.accept(buf.getAtIndex(PTypeIO.LE_LONG, i % elementCount)); + c.accept(buf.getAtIndex(VortexFormat.LE_LONG, i % elementCount)); } } } @@ -48,11 +48,11 @@ public long fold(long identity, LongBinaryOperator op) { long result = identity; if (n == elementCount) { for (long i = 0; i < n; i++) { - result = op.applyAsLong(result, buf.getAtIndex(PTypeIO.LE_LONG, i)); + result = op.applyAsLong(result, buf.getAtIndex(VortexFormat.LE_LONG, i)); } } else { for (long i = 0; i < n; i++) { - result = op.applyAsLong(result, buf.getAtIndex(PTypeIO.LE_LONG, i % elementCount)); + result = op.applyAsLong(result, buf.getAtIndex(VortexFormat.LE_LONG, i % elementCount)); } } return result; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedShortArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedShortArray.java index 29859cc6..7790cc63 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedShortArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/MaterializedShortArray.java @@ -3,7 +3,7 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.util.function.LongBinaryOperator; @@ -23,12 +23,12 @@ public MaterializedShortArray(DType dtype, long length, MemorySegment buffer) { @Override public short getShort(long i) { - return buffer.getAtIndex(PTypeIO.LE_SHORT, length == elementCount ? i : i % elementCount); + return buffer.getAtIndex(VortexFormat.LE_SHORT, length == elementCount ? i : i % elementCount); } @Override public int getInt(long i) { - short raw = buffer.getAtIndex(PTypeIO.LE_SHORT, length == elementCount ? i : i % elementCount); + short raw = buffer.getAtIndex(VortexFormat.LE_SHORT, length == elementCount ? i : i % elementCount); boolean unsigned = dtype instanceof DType.Primitive p && p.ptype() == PType.U16; return unsigned ? Short.toUnsignedInt(raw) : raw; } @@ -40,11 +40,11 @@ public long fold(long identity, LongBinaryOperator op) { long result = identity; if (n == elementCount) { for (long i = 0; i < n; i++) { - result = op.applyAsLong(result, buf.getAtIndex(PTypeIO.LE_SHORT, i)); + result = op.applyAsLong(result, buf.getAtIndex(VortexFormat.LE_SHORT, i)); } } else { for (long i = 0; i < n; i++) { - result = op.applyAsLong(result, buf.getAtIndex(PTypeIO.LE_SHORT, i % elementCount)); + result = op.applyAsLong(result, buf.getAtIndex(VortexFormat.LE_SHORT, i % elementCount)); } } return result; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/ShortArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/ShortArray.java index fe2babfc..e2f7f2cd 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/ShortArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/ShortArray.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -68,7 +68,7 @@ default MemorySegment materialize(SegmentAllocator arena) { long n = length(); MemorySegment dst = arena.allocate(n * 2L, 2); for (long i = 0; i < n; i++) { - dst.setAtIndex(PTypeIO.LE_SHORT, i, getShort(i)); + dst.setAtIndex(VortexFormat.LE_SHORT, i, getShort(i)); } return dst; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinArray.java index c659b123..ec7b79f4 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinArray.java @@ -3,7 +3,7 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; @@ -158,13 +158,13 @@ static OffsetMode toOffsetMode(VarBinArray src, SegmentAllocator arena) { } MemorySegment outBytes = arena.allocate(totalBytes > 0 ? totalBytes : 1); MemorySegment outOffsets = arena.allocate((n + 1) * Long.BYTES, Long.BYTES); - outOffsets.setAtIndex(PTypeIO.LE_LONG, 0, 0L); + outOffsets.setAtIndex(VortexFormat.LE_LONG, 0, 0L); long bytePos = 0; for (long i = 0; i < n; i++) { byte[] b = src.getBytes(i); MemorySegment.copy(MemorySegment.ofArray(b), 0, outBytes, bytePos, b.length); bytePos += b.length; - outOffsets.setAtIndex(PTypeIO.LE_LONG, i + 1, bytePos); + outOffsets.setAtIndex(VortexFormat.LE_LONG, i + 1, bytePos); } return new OffsetMode(src.dtype(), n, outBytes.asReadOnly(), outOffsets, PType.I64); } @@ -226,13 +226,13 @@ public void forEachByteLength(IntConsumer c) { long n = length; if (offsetsPtype == PType.I32 || offsetsPtype == PType.U32) { for (long i = 0; i < n; i++) { - c.accept(offsetsSegment.getAtIndex(PTypeIO.LE_INT, i + 1) - - offsetsSegment.getAtIndex(PTypeIO.LE_INT, i)); + c.accept(offsetsSegment.getAtIndex(VortexFormat.LE_INT, i + 1) + - offsetsSegment.getAtIndex(VortexFormat.LE_INT, i)); } } else { for (long i = 0; i < n; i++) { - c.accept((int) (offsetsSegment.getAtIndex(PTypeIO.LE_LONG, i + 1) - - offsetsSegment.getAtIndex(PTypeIO.LE_LONG, i))); + c.accept((int) (offsetsSegment.getAtIndex(VortexFormat.LE_LONG, i + 1) + - offsetsSegment.getAtIndex(VortexFormat.LE_LONG, i))); } } } @@ -252,9 +252,9 @@ public VarBinArray limited(long rows) { private long readOffset(long i) { if (offsetsPtype == PType.I32 || offsetsPtype == PType.U32) { - return offsetsSegment.getAtIndex(PTypeIO.LE_INT, i); + return offsetsSegment.getAtIndex(VortexFormat.LE_INT, i); } - return offsetsSegment.getAtIndex(PTypeIO.LE_LONG, i); + return offsetsSegment.getAtIndex(VortexFormat.LE_LONG, i); } } @@ -318,19 +318,19 @@ public VarBinArray limited(long rows) { private long dictReadCode(long i) { return switch (dictCodesPType) { case U8 -> Byte.toUnsignedLong(dictCodesSegs.get(ValueLayout.JAVA_BYTE, i)); - case U16 -> Short.toUnsignedLong(dictCodesSegs.getAtIndex(PTypeIO.LE_SHORT, i)); - case U32 -> Integer.toUnsignedLong(dictCodesSegs.getAtIndex(PTypeIO.LE_INT, i)); - case I32 -> dictCodesSegs.getAtIndex(PTypeIO.LE_INT, i); - case I64, U64 -> dictCodesSegs.getAtIndex(PTypeIO.LE_LONG, i); + case U16 -> Short.toUnsignedLong(dictCodesSegs.getAtIndex(VortexFormat.LE_SHORT, i)); + case U32 -> Integer.toUnsignedLong(dictCodesSegs.getAtIndex(VortexFormat.LE_INT, i)); + case I32 -> dictCodesSegs.getAtIndex(VortexFormat.LE_INT, i); + case I64, U64 -> dictCodesSegs.getAtIndex(VortexFormat.LE_LONG, i); default -> throw new VortexException("unsupported codes ptype: " + dictCodesPType); }; } private long dictReadOff(long i) { if (dictValOffPType == PType.I32 || dictValOffPType == PType.U32) { - return dictValOffsets.getAtIndex(PTypeIO.LE_INT, i); + return dictValOffsets.getAtIndex(VortexFormat.LE_INT, i); } - return dictValOffsets.getAtIndex(PTypeIO.LE_LONG, i); + return dictValOffsets.getAtIndex(VortexFormat.LE_LONG, i); } } @@ -495,19 +495,19 @@ public Optional segmentIfPresent() { @Override public int getByteLength(long i) { - return views.get(PTypeIO.LE_INT, i * VIEW_SIZE); + return views.get(VortexFormat.LE_INT, i * VIEW_SIZE); } @Override public byte[] getBytes(long i) { long viewOff = i * VIEW_SIZE; - int size = views.get(PTypeIO.LE_INT, viewOff); + int size = views.get(VortexFormat.LE_INT, viewOff); byte[] out = new byte[size]; if (size <= MAX_INLINED_SIZE) { MemorySegment.copy(views, viewOff + 4, MemorySegment.ofArray(out), 0, size); } else { - int bufferIndex = views.get(PTypeIO.LE_INT, viewOff + 8); - long srcOffset = Integer.toUnsignedLong(views.get(PTypeIO.LE_INT, viewOff + 12)); + int bufferIndex = views.get(VortexFormat.LE_INT, viewOff + 8); + long srcOffset = Integer.toUnsignedLong(views.get(VortexFormat.LE_INT, viewOff + 12)); MemorySegment.copy(dataBufs[bufferIndex], srcOffset, MemorySegment.ofArray(out), 0, size); } return out; @@ -522,7 +522,7 @@ public String getString(long i) { public void forEachByteLength(IntConsumer c) { long n = length; for (long i = 0; i < n; i++) { - c.accept(views.get(PTypeIO.LE_INT, i * VIEW_SIZE)); + c.accept(views.get(VortexFormat.LE_INT, i * VIEW_SIZE)); } } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java index 5345b39b..33c75d99 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpEncodingDecoder.java @@ -4,7 +4,7 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoALPMetadata; import io.github.dfa1.vortex.core.proto.ProtoPatchesMetadata; import io.github.dfa1.vortex.reader.array.Array; @@ -81,7 +81,7 @@ private static Array decodeF64(DecodeContext ctx, ProtoALPMetadata meta, int exp return new LazyAlpDoubleArray(ctx.dtype(), n, src, df, de); } // broadcast without patches: decode single encoded value → constant - double v = src.getAtIndex(PTypeIO.LE_LONG, 0) * df * de; + double v = src.getAtIndex(VortexFormat.LE_LONG, 0) * df * de; return new LazyConstantDoubleArray(ctx.dtype(), n, v); } @@ -89,11 +89,11 @@ private static Array decodeF64(DecodeContext ctx, ProtoALPMetadata meta, int exp MemorySegment buf = ctx.arena().allocate(n * 8, 8); if (srcCap == n) { for (long i = 0; i < n; i++) { - buf.setAtIndex(PTypeIO.LE_DOUBLE, i, src.getAtIndex(PTypeIO.LE_LONG, i) * df * de); + buf.setAtIndex(VortexFormat.LE_DOUBLE, i, src.getAtIndex(VortexFormat.LE_LONG, i) * df * de); } } else { for (long i = 0; i < n; i++) { - buf.setAtIndex(PTypeIO.LE_DOUBLE, i, src.getAtIndex(PTypeIO.LE_LONG, i % srcCap) * df * de); + buf.setAtIndex(VortexFormat.LE_DOUBLE, i, src.getAtIndex(VortexFormat.LE_LONG, i % srcCap) * df * de); } } applyPatches(ctx, meta.patches(), buf, 8); @@ -111,7 +111,7 @@ private static Array decodeF32(DecodeContext ctx, ProtoALPMetadata meta, int exp return new LazyAlpFloatArray(ctx.dtype(), n, src, df, de); } // broadcast without patches: decode single encoded value → constant - float v = src.getAtIndex(PTypeIO.LE_INT, 0) * df * de; + float v = src.getAtIndex(VortexFormat.LE_INT, 0) * df * de; return new LazyConstantFloatArray(ctx.dtype(), n, v); } @@ -119,11 +119,11 @@ private static Array decodeF32(DecodeContext ctx, ProtoALPMetadata meta, int exp MemorySegment buf = ctx.arena().allocate(n * 4, 4); if (srcCap == n) { for (long i = 0; i < n; i++) { - buf.setAtIndex(PTypeIO.LE_FLOAT, i, src.getAtIndex(PTypeIO.LE_INT, i) * df * de); + buf.setAtIndex(VortexFormat.LE_FLOAT, i, src.getAtIndex(VortexFormat.LE_INT, i) * df * de); } } else { for (long i = 0; i < n; i++) { - buf.setAtIndex(PTypeIO.LE_FLOAT, i, src.getAtIndex(PTypeIO.LE_INT, i % srcCap) * df * de); + buf.setAtIndex(VortexFormat.LE_FLOAT, i, src.getAtIndex(VortexFormat.LE_INT, i % srcCap) * df * de); } } applyPatches(ctx, meta.patches(), buf, 4); @@ -157,9 +157,9 @@ private static void applyPatches(DecodeContext ctx, ProtoPatchesMetadata pm, Mem private static long readUnsigned(MemorySegment seg, long off, PType ptype) { return switch (ptype) { case U8 -> Byte.toUnsignedLong(seg.get(ValueLayout.JAVA_BYTE, off)); - case U16 -> Short.toUnsignedLong(seg.get(PTypeIO.LE_SHORT, off)); - case U32 -> Integer.toUnsignedLong(seg.get(PTypeIO.LE_INT, off)); - case U64 -> seg.get(PTypeIO.LE_LONG, off); + case U16 -> Short.toUnsignedLong(seg.get(VortexFormat.LE_SHORT, off)); + case U32 -> Integer.toUnsignedLong(seg.get(VortexFormat.LE_INT, off)); + case U64 -> seg.get(VortexFormat.LE_LONG, off); default -> throw new VortexException(EncodingId.VORTEX_ALP, "non-unsigned patch index ptype " + ptype); }; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java index 55a83e3a..0afe9c9c 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/AlpRdEncodingDecoder.java @@ -4,7 +4,7 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoALPRDMetadata; import io.github.dfa1.vortex.core.proto.ProtoPatchesMetadata; import io.github.dfa1.vortex.reader.array.Array; @@ -102,7 +102,7 @@ private static Patches decodePatches(DecodeContext ctx, ProtoPatchesMetadata pm) long valCap = SegmentBroadcast.capacity(valSeg, 2); short[] leftValues = new short[(int) numPatches]; for (int j = 0; j < numPatches; j++) { - leftValues[j] = valSeg.getAtIndex(PTypeIO.LE_SHORT, j % valCap); + leftValues[j] = valSeg.getAtIndex(VortexFormat.LE_SHORT, j % valCap); } return new Patches(idxData, leftValues, offset); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java index 130d0147..fef2af69 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/BitpackedEncodingDecoder.java @@ -4,7 +4,7 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoBitPackedMetadata; import io.github.dfa1.vortex.core.proto.ProtoPatchesMetadata; import io.github.dfa1.vortex.reader.array.Array; @@ -253,15 +253,15 @@ private static void unpackLoop16(MemorySegment buf, int bitWidth, int offset, lo long hiMask = hiMasks[row]; long laneOff = 0L; for (int lane = 0; lane < lanes; lane++, laneOff += 2L) { - long lo = (Short.toUnsignedLong(buf.get(PTypeIO.LE_SHORT, wordBase + laneOff)) >>> shift) & loMask; - long hi = Short.toUnsignedLong(buf.get(PTypeIO.LE_SHORT, hiBase + laneOff)) & hiMask; - out.set(PTypeIO.LE_SHORT, outBase + laneOff, (short) (lo | (hi << curr))); + long lo = (Short.toUnsignedLong(buf.get(VortexFormat.LE_SHORT, wordBase + laneOff)) >>> shift) & loMask; + long hi = Short.toUnsignedLong(buf.get(VortexFormat.LE_SHORT, hiBase + laneOff)) & hiMask; + out.set(VortexFormat.LE_SHORT, outBase + laneOff, (short) (lo | (hi << curr))); } } else { long laneOff = 0L; for (int lane = 0; lane < lanes; lane++, laneOff += 2L) { - out.set(PTypeIO.LE_SHORT, outBase + laneOff, - (short) ((Short.toUnsignedLong(buf.get(PTypeIO.LE_SHORT, wordBase + laneOff)) >>> shift) & bitMask)); + out.set(VortexFormat.LE_SHORT, outBase + laneOff, + (short) ((Short.toUnsignedLong(buf.get(VortexFormat.LE_SHORT, wordBase + laneOff)) >>> shift) & bitMask)); } } } @@ -282,16 +282,16 @@ private static void unpackLoop16(MemorySegment buf, int bitWidth, int offset, lo if (logicalIdx < 0 || logicalIdx >= rowCount) { continue; } - long src = Short.toUnsignedLong(buf.get(PTypeIO.LE_SHORT, wordBase + (long) lane * 2)); + long src = Short.toUnsignedLong(buf.get(VortexFormat.LE_SHORT, wordBase + (long) lane * 2)); long value; if (rem > 0) { long lo = (src >>> shift) & loMask; - long hi = Short.toUnsignedLong(buf.get(PTypeIO.LE_SHORT, hiBase + (long) lane * 2)) & hiMask; + long hi = Short.toUnsignedLong(buf.get(VortexFormat.LE_SHORT, hiBase + (long) lane * 2)) & hiMask; value = lo | (hi << curr); } else { value = (src >>> shift) & bitMask; } - out.set(PTypeIO.LE_SHORT, (long) logicalIdx * 2, (short) value); + out.set(VortexFormat.LE_SHORT, (long) logicalIdx * 2, (short) value); } } } @@ -334,15 +334,15 @@ private static void unpackLoop32(MemorySegment buf, int bitWidth, int offset, lo long hiMask = hiMasks[row]; long laneOff = 0L; for (int lane = 0; lane < lanes; lane++, laneOff += 4L) { - long lo = (Integer.toUnsignedLong(buf.get(PTypeIO.LE_INT, wordBase + laneOff)) >>> shift) & loMask; - long hi = Integer.toUnsignedLong(buf.get(PTypeIO.LE_INT, hiBase + laneOff)) & hiMask; - out.set(PTypeIO.LE_INT, outBase + laneOff, (int) (lo | (hi << curr))); + long lo = (Integer.toUnsignedLong(buf.get(VortexFormat.LE_INT, wordBase + laneOff)) >>> shift) & loMask; + long hi = Integer.toUnsignedLong(buf.get(VortexFormat.LE_INT, hiBase + laneOff)) & hiMask; + out.set(VortexFormat.LE_INT, outBase + laneOff, (int) (lo | (hi << curr))); } } else { long laneOff = 0L; for (int lane = 0; lane < lanes; lane++, laneOff += 4L) { - out.set(PTypeIO.LE_INT, outBase + laneOff, - (int) ((Integer.toUnsignedLong(buf.get(PTypeIO.LE_INT, wordBase + laneOff)) >>> shift) & bitMask)); + out.set(VortexFormat.LE_INT, outBase + laneOff, + (int) ((Integer.toUnsignedLong(buf.get(VortexFormat.LE_INT, wordBase + laneOff)) >>> shift) & bitMask)); } } } @@ -363,16 +363,16 @@ private static void unpackLoop32(MemorySegment buf, int bitWidth, int offset, lo if (logicalIdx < 0 || logicalIdx >= rowCount) { continue; } - long src = Integer.toUnsignedLong(buf.get(PTypeIO.LE_INT, wordBase + (long) lane * 4)); + long src = Integer.toUnsignedLong(buf.get(VortexFormat.LE_INT, wordBase + (long) lane * 4)); long value; if (rem > 0) { long lo = (src >>> shift) & loMask; - long hi = Integer.toUnsignedLong(buf.get(PTypeIO.LE_INT, hiBase + (long) lane * 4)) & hiMask; + long hi = Integer.toUnsignedLong(buf.get(VortexFormat.LE_INT, hiBase + (long) lane * 4)) & hiMask; value = lo | (hi << curr); } else { value = (src >>> shift) & bitMask; } - out.set(PTypeIO.LE_INT, (long) logicalIdx * 4, (int) value); + out.set(VortexFormat.LE_INT, (long) logicalIdx * 4, (int) value); } } } @@ -415,15 +415,15 @@ private static void unpackLoop64(MemorySegment buf, int bitWidth, int offset, lo long hiMask = hiMasks[row]; long laneOff = 0L; for (int lane = 0; lane < lanes; lane++, laneOff += 8L) { - long lo = (buf.get(PTypeIO.LE_LONG, wordBase + laneOff) >>> shift) & loMask; - long hi = buf.get(PTypeIO.LE_LONG, hiBase + laneOff) & hiMask; - out.set(PTypeIO.LE_LONG, outBase + laneOff, lo | (hi << curr)); + long lo = (buf.get(VortexFormat.LE_LONG, wordBase + laneOff) >>> shift) & loMask; + long hi = buf.get(VortexFormat.LE_LONG, hiBase + laneOff) & hiMask; + out.set(VortexFormat.LE_LONG, outBase + laneOff, lo | (hi << curr)); } } else { long laneOff = 0L; for (int lane = 0; lane < lanes; lane++, laneOff += 8L) { - out.set(PTypeIO.LE_LONG, outBase + laneOff, - (buf.get(PTypeIO.LE_LONG, wordBase + laneOff) >>> shift) & bitMask); + out.set(VortexFormat.LE_LONG, outBase + laneOff, + (buf.get(VortexFormat.LE_LONG, wordBase + laneOff) >>> shift) & bitMask); } } } @@ -444,16 +444,16 @@ private static void unpackLoop64(MemorySegment buf, int bitWidth, int offset, lo if (logicalIdx < 0 || logicalIdx >= rowCount) { continue; } - long src = buf.get(PTypeIO.LE_LONG, wordBase + (long) lane * 8); + long src = buf.get(VortexFormat.LE_LONG, wordBase + (long) lane * 8); long value; if (rem > 0) { long lo = (src >>> shift) & loMask; - long hi = buf.get(PTypeIO.LE_LONG, hiBase + (long) lane * 8) & hiMask; + long hi = buf.get(VortexFormat.LE_LONG, hiBase + (long) lane * 8) & hiMask; value = lo | (hi << curr); } else { value = (src >>> shift) & bitMask; } - out.set(PTypeIO.LE_LONG, (long) logicalIdx * 8, value); + out.set(VortexFormat.LE_LONG, (long) logicalIdx * 8, value); } } } @@ -488,9 +488,9 @@ private static void applyPatches(DecodeContext ctx, ProtoPatchesMetadata pm, private static long readUnsignedIdx(MemorySegment seg, long off, PType ptype) { return switch (ptype) { case U8 -> Byte.toUnsignedLong(seg.get(ValueLayout.JAVA_BYTE, off)); - case U16 -> Short.toUnsignedLong(seg.get(PTypeIO.LE_SHORT, off)); - case U32 -> Integer.toUnsignedLong(seg.get(PTypeIO.LE_INT, off)); - case U64 -> seg.get(PTypeIO.LE_LONG, off); + case U16 -> Short.toUnsignedLong(seg.get(VortexFormat.LE_SHORT, off)); + case U32 -> Integer.toUnsignedLong(seg.get(VortexFormat.LE_INT, off)); + case U64 -> seg.get(VortexFormat.LE_LONG, off); default -> throw new VortexException(EncodingId.FASTLANES_BITPACKED, "non-unsigned patch index ptype " + ptype); }; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ChunkedEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ChunkedEncodingDecoder.java index defa94e7..e3fd638d 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ChunkedEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ChunkedEncodingDecoder.java @@ -15,16 +15,14 @@ import io.github.dfa1.vortex.reader.array.StructArray; import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.nio.ByteOrder; import java.util.ArrayList; import java.util.List; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_LONG; + /// Read-only decoder for `vortex.chunked`. public final class ChunkedEncodingDecoder implements EncodingDecoder { - private static final ValueLayout.OfLong LE_LONG = - ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); /// Public no-arg constructor required by [java.util.ServiceLoader]. public ChunkedEncodingDecoder() { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoder.java index f071e741..fba0d2b7 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ConstantEncodingDecoder.java @@ -4,7 +4,7 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.LazyConstantBoolArray; @@ -140,7 +140,7 @@ private static Array decodeString(DecodeContext ctx, ProtoScalarValue scalar, DT MemorySegment offsetsSeg = ctx.arena().allocate((n + 1) * 4L, 4); for (long i = 0; i <= n; i++) { - offsetsSeg.setAtIndex(PTypeIO.LE_INT, i, (int) (i * strLen)); + offsetsSeg.setAtIndex(VortexFormat.LE_INT, i, (int) (i * strLen)); } return new VarBinArray.OffsetMode(dtype, n, bytesSeg.asReadOnly(), offsetsSeg.asReadOnly(), PType.I32); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java index 6a6f26f9..c7c9f4aa 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoder.java @@ -6,7 +6,7 @@ import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.compute.FastLanes; import io.github.dfa1.vortex.core.compute.PrimitiveArrays; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoDeltaMetadata; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.MaterializedByteArray; @@ -127,11 +127,11 @@ private static long[] readLongs(MemorySegment buf, int count, PType ptype) { out[i] = switch (ptype) { case I8 -> buf.get(ValueLayout.JAVA_BYTE, off); case U8 -> Byte.toUnsignedLong(buf.get(ValueLayout.JAVA_BYTE, off)); - case I16 -> buf.get(PTypeIO.LE_SHORT, off); - case U16 -> Short.toUnsignedLong(buf.get(PTypeIO.LE_SHORT, off)); - case I32 -> buf.get(PTypeIO.LE_INT, off); - case U32 -> Integer.toUnsignedLong(buf.get(PTypeIO.LE_INT, off)); - case I64, U64 -> buf.get(PTypeIO.LE_LONG, off); + case I16 -> buf.get(VortexFormat.LE_SHORT, off); + case U16 -> Short.toUnsignedLong(buf.get(VortexFormat.LE_SHORT, off)); + case I32 -> buf.get(VortexFormat.LE_INT, off); + case U32 -> Integer.toUnsignedLong(buf.get(VortexFormat.LE_INT, off)); + case I64, U64 -> buf.get(VortexFormat.LE_LONG, off); default -> throw new VortexException(EncodingId.FASTLANES_DELTA, "unsupported ptype: " + ptype); }; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java index 43ac689e..b1fd8052 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/DictEncodingDecoder.java @@ -4,7 +4,7 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoDictMetadata; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.MaterializedByteArray; @@ -164,12 +164,12 @@ static void expandU8(MemorySegment codes, MemorySegment values, MemorySegment ou if (fast) { for (long i = 0; i < rowCount; i++) { long code = Byte.toUnsignedLong(codes.get(ValueLayout.JAVA_BYTE, i)); - out.setAtIndex(PTypeIO.LE_LONG, i, values.getAtIndex(PTypeIO.LE_LONG, code)); + out.setAtIndex(VortexFormat.LE_LONG, i, values.getAtIndex(VortexFormat.LE_LONG, code)); } } else { for (long i = 0; i < rowCount; i++) { long code = Byte.toUnsignedLong(codes.get(ValueLayout.JAVA_BYTE, i % codesCap)); - out.setAtIndex(PTypeIO.LE_LONG, i, values.getAtIndex(PTypeIO.LE_LONG, code % valuesCap)); + out.setAtIndex(VortexFormat.LE_LONG, i, values.getAtIndex(VortexFormat.LE_LONG, code % valuesCap)); } } } @@ -177,12 +177,12 @@ static void expandU8(MemorySegment codes, MemorySegment values, MemorySegment ou if (fast) { for (long i = 0; i < rowCount; i++) { long code = Byte.toUnsignedLong(codes.get(ValueLayout.JAVA_BYTE, i)); - out.setAtIndex(PTypeIO.LE_INT, i, values.getAtIndex(PTypeIO.LE_INT, code)); + out.setAtIndex(VortexFormat.LE_INT, i, values.getAtIndex(VortexFormat.LE_INT, code)); } } else { for (long i = 0; i < rowCount; i++) { long code = Byte.toUnsignedLong(codes.get(ValueLayout.JAVA_BYTE, i % codesCap)); - out.setAtIndex(PTypeIO.LE_INT, i, values.getAtIndex(PTypeIO.LE_INT, code % valuesCap)); + out.setAtIndex(VortexFormat.LE_INT, i, values.getAtIndex(VortexFormat.LE_INT, code % valuesCap)); } } } @@ -190,12 +190,12 @@ static void expandU8(MemorySegment codes, MemorySegment values, MemorySegment ou if (fast) { for (long i = 0; i < rowCount; i++) { long code = Byte.toUnsignedLong(codes.get(ValueLayout.JAVA_BYTE, i)); - out.setAtIndex(PTypeIO.LE_SHORT, i, values.getAtIndex(PTypeIO.LE_SHORT, code)); + out.setAtIndex(VortexFormat.LE_SHORT, i, values.getAtIndex(VortexFormat.LE_SHORT, code)); } } else { for (long i = 0; i < rowCount; i++) { long code = Byte.toUnsignedLong(codes.get(ValueLayout.JAVA_BYTE, i % codesCap)); - out.setAtIndex(PTypeIO.LE_SHORT, i, values.getAtIndex(PTypeIO.LE_SHORT, code % valuesCap)); + out.setAtIndex(VortexFormat.LE_SHORT, i, values.getAtIndex(VortexFormat.LE_SHORT, code % valuesCap)); } } } @@ -236,51 +236,51 @@ static void expandU16(MemorySegment codes, MemorySegment values, MemorySegment o case 8 -> { if (fast) { for (long i = 0; i < rowCount; i++) { - long code = Short.toUnsignedLong(codes.get(PTypeIO.LE_SHORT, i * 2)); - out.setAtIndex(PTypeIO.LE_LONG, i, values.getAtIndex(PTypeIO.LE_LONG, code)); + long code = Short.toUnsignedLong(codes.get(VortexFormat.LE_SHORT, i * 2)); + out.setAtIndex(VortexFormat.LE_LONG, i, values.getAtIndex(VortexFormat.LE_LONG, code)); } } else { for (long i = 0; i < rowCount; i++) { - long code = Short.toUnsignedLong(codes.get(PTypeIO.LE_SHORT, (i % codesCap) * 2)); - out.setAtIndex(PTypeIO.LE_LONG, i, values.getAtIndex(PTypeIO.LE_LONG, code % valuesCap)); + long code = Short.toUnsignedLong(codes.get(VortexFormat.LE_SHORT, (i % codesCap) * 2)); + out.setAtIndex(VortexFormat.LE_LONG, i, values.getAtIndex(VortexFormat.LE_LONG, code % valuesCap)); } } } case 4 -> { if (fast) { for (long i = 0; i < rowCount; i++) { - long code = Short.toUnsignedLong(codes.get(PTypeIO.LE_SHORT, i * 2)); - out.setAtIndex(PTypeIO.LE_INT, i, values.getAtIndex(PTypeIO.LE_INT, code)); + long code = Short.toUnsignedLong(codes.get(VortexFormat.LE_SHORT, i * 2)); + out.setAtIndex(VortexFormat.LE_INT, i, values.getAtIndex(VortexFormat.LE_INT, code)); } } else { for (long i = 0; i < rowCount; i++) { - long code = Short.toUnsignedLong(codes.get(PTypeIO.LE_SHORT, (i % codesCap) * 2)); - out.setAtIndex(PTypeIO.LE_INT, i, values.getAtIndex(PTypeIO.LE_INT, code % valuesCap)); + long code = Short.toUnsignedLong(codes.get(VortexFormat.LE_SHORT, (i % codesCap) * 2)); + out.setAtIndex(VortexFormat.LE_INT, i, values.getAtIndex(VortexFormat.LE_INT, code % valuesCap)); } } } case 2 -> { if (fast) { for (long i = 0; i < rowCount; i++) { - long code = Short.toUnsignedLong(codes.get(PTypeIO.LE_SHORT, i * 2)); - out.setAtIndex(PTypeIO.LE_SHORT, i, values.getAtIndex(PTypeIO.LE_SHORT, code)); + long code = Short.toUnsignedLong(codes.get(VortexFormat.LE_SHORT, i * 2)); + out.setAtIndex(VortexFormat.LE_SHORT, i, values.getAtIndex(VortexFormat.LE_SHORT, code)); } } else { for (long i = 0; i < rowCount; i++) { - long code = Short.toUnsignedLong(codes.get(PTypeIO.LE_SHORT, (i % codesCap) * 2)); - out.setAtIndex(PTypeIO.LE_SHORT, i, values.getAtIndex(PTypeIO.LE_SHORT, code % valuesCap)); + long code = Short.toUnsignedLong(codes.get(VortexFormat.LE_SHORT, (i % codesCap) * 2)); + out.setAtIndex(VortexFormat.LE_SHORT, i, values.getAtIndex(VortexFormat.LE_SHORT, code % valuesCap)); } } } case 1 -> { if (fast) { for (long i = 0; i < rowCount; i++) { - long code = Short.toUnsignedLong(codes.get(PTypeIO.LE_SHORT, i * 2)); + long code = Short.toUnsignedLong(codes.get(VortexFormat.LE_SHORT, i * 2)); out.set(ValueLayout.JAVA_BYTE, i, values.get(ValueLayout.JAVA_BYTE, code)); } } else { for (long i = 0; i < rowCount; i++) { - long code = Short.toUnsignedLong(codes.get(PTypeIO.LE_SHORT, (i % codesCap) * 2)); + long code = Short.toUnsignedLong(codes.get(VortexFormat.LE_SHORT, (i % codesCap) * 2)); out.set(ValueLayout.JAVA_BYTE, i, values.get(ValueLayout.JAVA_BYTE, code % valuesCap)); } } @@ -288,12 +288,12 @@ static void expandU16(MemorySegment codes, MemorySegment values, MemorySegment o default -> { if (fast) { for (long i = 0, outOff = 0; i < rowCount; i++, outOff += elemSize) { - long code = Short.toUnsignedLong(codes.get(PTypeIO.LE_SHORT, i * 2)); + long code = Short.toUnsignedLong(codes.get(VortexFormat.LE_SHORT, i * 2)); MemorySegment.copy(values, code * elemSize, out, outOff, elemSize); } } else { for (long i = 0, outOff = 0; i < rowCount; i++, outOff += elemSize) { - long code = Short.toUnsignedLong(codes.get(PTypeIO.LE_SHORT, (i % codesCap) * 2)); + long code = Short.toUnsignedLong(codes.get(VortexFormat.LE_SHORT, (i % codesCap) * 2)); MemorySegment.copy(values, (code % valuesCap) * elemSize, out, outOff, elemSize); } } @@ -309,51 +309,51 @@ static void expandU32(MemorySegment codes, MemorySegment values, MemorySegment o case 8 -> { if (fast) { for (long i = 0; i < rowCount; i++) { - long code = Integer.toUnsignedLong(codes.get(PTypeIO.LE_INT, i * 4)); - out.setAtIndex(PTypeIO.LE_LONG, i, values.getAtIndex(PTypeIO.LE_LONG, code)); + long code = Integer.toUnsignedLong(codes.get(VortexFormat.LE_INT, i * 4)); + out.setAtIndex(VortexFormat.LE_LONG, i, values.getAtIndex(VortexFormat.LE_LONG, code)); } } else { for (long i = 0; i < rowCount; i++) { - long code = Integer.toUnsignedLong(codes.get(PTypeIO.LE_INT, (i % codesCap) * 4)); - out.setAtIndex(PTypeIO.LE_LONG, i, values.getAtIndex(PTypeIO.LE_LONG, code % valuesCap)); + long code = Integer.toUnsignedLong(codes.get(VortexFormat.LE_INT, (i % codesCap) * 4)); + out.setAtIndex(VortexFormat.LE_LONG, i, values.getAtIndex(VortexFormat.LE_LONG, code % valuesCap)); } } } case 4 -> { if (fast) { for (long i = 0; i < rowCount; i++) { - long code = Integer.toUnsignedLong(codes.get(PTypeIO.LE_INT, i * 4)); - out.setAtIndex(PTypeIO.LE_INT, i, values.getAtIndex(PTypeIO.LE_INT, code)); + long code = Integer.toUnsignedLong(codes.get(VortexFormat.LE_INT, i * 4)); + out.setAtIndex(VortexFormat.LE_INT, i, values.getAtIndex(VortexFormat.LE_INT, code)); } } else { for (long i = 0; i < rowCount; i++) { - long code = Integer.toUnsignedLong(codes.get(PTypeIO.LE_INT, (i % codesCap) * 4)); - out.setAtIndex(PTypeIO.LE_INT, i, values.getAtIndex(PTypeIO.LE_INT, code % valuesCap)); + long code = Integer.toUnsignedLong(codes.get(VortexFormat.LE_INT, (i % codesCap) * 4)); + out.setAtIndex(VortexFormat.LE_INT, i, values.getAtIndex(VortexFormat.LE_INT, code % valuesCap)); } } } case 2 -> { if (fast) { for (long i = 0; i < rowCount; i++) { - long code = Integer.toUnsignedLong(codes.get(PTypeIO.LE_INT, i * 4)); - out.setAtIndex(PTypeIO.LE_SHORT, i, values.getAtIndex(PTypeIO.LE_SHORT, code)); + long code = Integer.toUnsignedLong(codes.get(VortexFormat.LE_INT, i * 4)); + out.setAtIndex(VortexFormat.LE_SHORT, i, values.getAtIndex(VortexFormat.LE_SHORT, code)); } } else { for (long i = 0; i < rowCount; i++) { - long code = Integer.toUnsignedLong(codes.get(PTypeIO.LE_INT, (i % codesCap) * 4)); - out.setAtIndex(PTypeIO.LE_SHORT, i, values.getAtIndex(PTypeIO.LE_SHORT, code % valuesCap)); + long code = Integer.toUnsignedLong(codes.get(VortexFormat.LE_INT, (i % codesCap) * 4)); + out.setAtIndex(VortexFormat.LE_SHORT, i, values.getAtIndex(VortexFormat.LE_SHORT, code % valuesCap)); } } } case 1 -> { if (fast) { for (long i = 0; i < rowCount; i++) { - long code = Integer.toUnsignedLong(codes.get(PTypeIO.LE_INT, i * 4)); + long code = Integer.toUnsignedLong(codes.get(VortexFormat.LE_INT, i * 4)); out.set(ValueLayout.JAVA_BYTE, i, values.get(ValueLayout.JAVA_BYTE, code)); } } else { for (long i = 0; i < rowCount; i++) { - long code = Integer.toUnsignedLong(codes.get(PTypeIO.LE_INT, (i % codesCap) * 4)); + long code = Integer.toUnsignedLong(codes.get(VortexFormat.LE_INT, (i % codesCap) * 4)); out.set(ValueLayout.JAVA_BYTE, i, values.get(ValueLayout.JAVA_BYTE, code % valuesCap)); } } @@ -361,12 +361,12 @@ static void expandU32(MemorySegment codes, MemorySegment values, MemorySegment o default -> { if (fast) { for (long i = 0, outOff = 0; i < rowCount; i++, outOff += elemSize) { - long code = Integer.toUnsignedLong(codes.get(PTypeIO.LE_INT, i * 4)); + long code = Integer.toUnsignedLong(codes.get(VortexFormat.LE_INT, i * 4)); MemorySegment.copy(values, code * elemSize, out, outOff, elemSize); } } else { for (long i = 0, outOff = 0; i < rowCount; i++, outOff += elemSize) { - long code = Integer.toUnsignedLong(codes.get(PTypeIO.LE_INT, (i % codesCap) * 4)); + long code = Integer.toUnsignedLong(codes.get(VortexFormat.LE_INT, (i % codesCap) * 4)); MemorySegment.copy(values, (code % valuesCap) * elemSize, out, outOff, elemSize); } } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoder.java index 6e37fa9c..6733ab88 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/FsstEncodingDecoder.java @@ -6,7 +6,7 @@ import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.VarBinArray; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoFSSTMetadata; import java.io.IOException; @@ -62,7 +62,7 @@ public Array decode(DecodeContext ctx) { MemorySegment outBytes = ctx.arena().allocate(totalUncompressed); MemorySegment outOffsets = ctx.arena().allocate((n + 1) * 4L, 4); - outOffsets.setAtIndex(PTypeIO.LE_INT, 0, 0); + outOffsets.setAtIndex(VortexFormat.LE_INT, 0, 0); long outPos = 0L; for (long i = 0; i < n; i++) { @@ -70,7 +70,7 @@ public Array decode(DecodeContext ctx) { long cEnd = readUnsigned(codesOffsetsSeg, (i + 1) % codesOffCap, codesOffPType); outPos = decompressString(compressedBytes, symbolsBuf, symbolLensBuf, cStart, cEnd, outBytes, outPos); - outOffsets.setAtIndex(PTypeIO.LE_INT, i + 1, (int) outPos); + outOffsets.setAtIndex(VortexFormat.LE_INT, i + 1, (int) outPos); } return new VarBinArray.OffsetMode(ctx.dtype(), n, outBytes.asReadOnly(), outOffsets.asReadOnly(), PType.I32); @@ -86,7 +86,7 @@ private static long decompressString( out.set(ValueLayout.JAVA_BYTE, outPos++, compressed.get(ValueLayout.JAVA_BYTE, ++j)); } else { int symLen = Byte.toUnsignedInt(symLens.get(ValueLayout.JAVA_BYTE, b)); - long sym = symbols.getAtIndex(PTypeIO.LE_LONG, b); + long sym = symbols.getAtIndex(VortexFormat.LE_LONG, b); for (int k = 0; k < symLen; k++) { out.set(ValueLayout.JAVA_BYTE, outPos++, (byte) (sym >>> (k * 8))); } @@ -98,10 +98,10 @@ private static long decompressString( private static long readUnsigned(MemorySegment seg, long idx, PType ptype) { return switch (ptype) { case U8 -> Byte.toUnsignedLong(seg.get(ValueLayout.JAVA_BYTE, idx)); - case U16 -> Short.toUnsignedLong(seg.get(PTypeIO.LE_SHORT, idx * 2)); - case U32 -> Integer.toUnsignedLong(seg.getAtIndex(PTypeIO.LE_INT, idx)); - case I32 -> seg.getAtIndex(PTypeIO.LE_INT, idx); - case I64, U64 -> seg.getAtIndex(PTypeIO.LE_LONG, idx); + case U16 -> Short.toUnsignedLong(seg.get(VortexFormat.LE_SHORT, idx * 2)); + case U32 -> Integer.toUnsignedLong(seg.getAtIndex(VortexFormat.LE_INT, idx)); + case I32 -> seg.getAtIndex(VortexFormat.LE_INT, idx); + case I64, U64 -> seg.getAtIndex(VortexFormat.LE_LONG, idx); default -> throw new VortexException(EncodingId.VORTEX_FSST, "unsupported ptype " + ptype); }; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoder.java index d1f16d29..aa2a0c28 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PatchedEncodingDecoder.java @@ -4,7 +4,7 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoPatchedMetadata; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.MaterializedByteArray; @@ -99,13 +99,13 @@ private static void applyPatches( long valCap = SegmentBroadcast.capacity(patchValues, elemBytes); for (long chunk = 0; chunk < nChunks; chunk++) { long start = Integer.toUnsignedLong( - laneOffsets.getAtIndex(PTypeIO.LE_INT, (chunk * nLanes) % laneCap)); + laneOffsets.getAtIndex(VortexFormat.LE_INT, (chunk * nLanes) % laneCap)); long stop = Integer.toUnsignedLong( - laneOffsets.getAtIndex(PTypeIO.LE_INT, (chunk * nLanes + nLanes) % laneCap)); + laneOffsets.getAtIndex(VortexFormat.LE_INT, (chunk * nLanes + nLanes) % laneCap)); for (long i = start; i < stop; i++) { long physicalIdx = chunk * 1024 - + Short.toUnsignedLong(patchIndices.getAtIndex(PTypeIO.LE_SHORT, i % idxCap)); + + Short.toUnsignedLong(patchIndices.getAtIndex(VortexFormat.LE_SHORT, i % idxCap)); if (physicalIdx < offset || physicalIdx >= offset + n) { continue; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PcoEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PcoEncodingDecoder.java index d4f6594e..9aaae993 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PcoEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PcoEncodingDecoder.java @@ -19,8 +19,8 @@ import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; -import java.lang.foreign.ValueLayout; -import java.nio.ByteOrder; + +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_LONG; /// Read-only decoder for `vortex.pco` — port of pcodec. public final class PcoEncodingDecoder implements EncodingDecoder { @@ -30,8 +30,6 @@ public final class PcoEncodingDecoder implements EncodingDecoder { static final int BITS_TO_ENCODE_OFFSET_BITS_32 = 6; static final int BITS_TO_ENCODE_OFFSET_BITS_16 = 5; - private static final ValueLayout.OfLong LE_LONG = - ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); /// Public no-arg constructor required by [java.util.ServiceLoader]. public PcoEncodingDecoder() { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PcoTansDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PcoTansDecoder.java index 3adecf70..2a6cc242 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PcoTansDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/PcoTansDecoder.java @@ -4,8 +4,8 @@ import io.github.dfa1.vortex.core.model.EncodingId; import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.nio.ByteOrder; + +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_LONG; /// 4-way interleaved tANS decoder for one pco latent variable. /// @@ -15,8 +15,6 @@ public final class PcoTansDecoder { public static final int BATCH_N = 256; public static final int ANS_INTERLEAVING = 4; - private static final ValueLayout.OfLong LE_LONG = - ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); // All arrays indexed by state index in [0, tableSize). private final int[] nextStateIdxBase; // = (symbolXs[sym] << bitsToRead) - tableSize private final int[] bitsToRead; // bits consumed from bit stream per ANS step diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java index f8fd3aa0..87956b79 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RleEncodingDecoder.java @@ -4,7 +4,7 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoRLEMetadata; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.BoolArray; @@ -129,8 +129,8 @@ private static long[] readLongs(MemorySegment buf, int count, PType ptype) { for (int i = 0; i < count; i++) { long off = (i % cap) * elemSize; out[i] = switch (ptype) { - case I64 -> buf.get(PTypeIO.LE_LONG, off); - case U64 -> buf.get(PTypeIO.LE_LONG, off); + case I64 -> buf.get(VortexFormat.LE_LONG, off); + case U64 -> buf.get(VortexFormat.LE_LONG, off); default -> throw new VortexException(EncodingId.FASTLANES_RLE, "expected I64/U64, got " + ptype); }; } @@ -143,7 +143,7 @@ private static int[] readInts(MemorySegment buf, int count, PType ptype) { long cap = SegmentBroadcast.capacity(buf, elemSize); for (int i = 0; i < count; i++) { long off = (i % cap) * elemSize; - out[i] = buf.get(PTypeIO.LE_INT, off); + out[i] = buf.get(VortexFormat.LE_INT, off); } return out; } @@ -154,7 +154,7 @@ private static short[] readShorts(MemorySegment buf, int count, PType ptype) { long cap = SegmentBroadcast.capacity(buf, elemSize); for (int i = 0; i < count; i++) { long off = (i % cap) * elemSize; - out[i] = buf.get(PTypeIO.LE_SHORT, off); + out[i] = buf.get(VortexFormat.LE_SHORT, off); } return out; } @@ -180,7 +180,7 @@ private static int[] readIndices(MemorySegment buf, int count, PType indicesPtyp } case U16 -> { for (int i = 0; i < count; i++) { - out[i] = Short.toUnsignedInt(buf.get(PTypeIO.LE_SHORT, (i % cap) * 2)); + out[i] = Short.toUnsignedInt(buf.get(VortexFormat.LE_SHORT, (i % cap) * 2)); } } default -> @@ -197,9 +197,9 @@ private static long[] readUnsignedLongs(MemorySegment buf, int count, PType ptyp long off = (i % cap) * elemSize; out[i] = switch (ptype) { case U8 -> Byte.toUnsignedLong(buf.get(ValueLayout.JAVA_BYTE, off)); - case U16 -> Short.toUnsignedLong(buf.get(PTypeIO.LE_SHORT, off)); - case U32 -> Integer.toUnsignedLong(buf.get(PTypeIO.LE_INT, off)); - case U64 -> buf.get(PTypeIO.LE_LONG, off); + case U16 -> Short.toUnsignedLong(buf.get(VortexFormat.LE_SHORT, off)); + case U32 -> Integer.toUnsignedLong(buf.get(VortexFormat.LE_INT, off)); + case U64 -> buf.get(VortexFormat.LE_LONG, off); default -> throw new VortexException(EncodingId.FASTLANES_RLE, "unsupported offsets ptype: " + ptype); }; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java index 2efe4925..673d4b53 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/RunEndEncodingDecoder.java @@ -4,7 +4,7 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoRunEndMetadata; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.BoolArray; @@ -118,7 +118,7 @@ private static Array expandStrings( MemorySegment outBytes = arena.allocate(totalBytes > 0 ? totalBytes : 1); MemorySegment outOffsets = arena.allocate((n + 1) * 4L, 4); - outOffsets.setAtIndex(PTypeIO.LE_INT, 0, 0); + outOffsets.setAtIndex(VortexFormat.LE_INT, 0, 0); long bytePos = 0; long outIdx = 0; @@ -136,7 +136,7 @@ private static Array expandStrings( MemorySegment.copy(valBytes, strStart, outBytes, bytePos, strLen); bytePos += strLen; } - outOffsets.setAtIndex(PTypeIO.LE_INT, outIdx + 1, (int) bytePos); + outOffsets.setAtIndex(VortexFormat.LE_INT, outIdx + 1, (int) bytePos); } } logicalPos = runEnd; @@ -148,17 +148,17 @@ private static Array expandStrings( private static long readUnsigned(MemorySegment seg, long i, PType ptype) { return switch (ptype) { case U8 -> Byte.toUnsignedLong(seg.get(ValueLayout.JAVA_BYTE, i)); - case U16 -> Short.toUnsignedLong(seg.get(PTypeIO.LE_SHORT, i * 2)); - case U32 -> Integer.toUnsignedLong(seg.get(PTypeIO.LE_INT, i * 4)); - case U64 -> seg.get(PTypeIO.LE_LONG, i * 8); + case U16 -> Short.toUnsignedLong(seg.get(VortexFormat.LE_SHORT, i * 2)); + case U32 -> Integer.toUnsignedLong(seg.get(VortexFormat.LE_INT, i * 4)); + case U64 -> seg.get(VortexFormat.LE_LONG, i * 8); default -> throw new VortexException(EncodingId.VORTEX_RUNEND, "non-unsigned ends ptype " + ptype); }; } private static long readVarBinOffset(MemorySegment seg, long i, PType ptype) { return switch (ptype) { - case I32, U32 -> Integer.toUnsignedLong(seg.getAtIndex(PTypeIO.LE_INT, i)); - case I64, U64 -> seg.getAtIndex(PTypeIO.LE_LONG, i); + case I32, U32 -> Integer.toUnsignedLong(seg.getAtIndex(VortexFormat.LE_INT, i)); + case I64, U64 -> seg.getAtIndex(VortexFormat.LE_LONG, i); default -> throw new VortexException(EncodingId.VORTEX_RUNEND, "unsupported offset ptype " + ptype); }; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SequenceEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SequenceEncodingDecoder.java index 63b00bf4..1b365cfa 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SequenceEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SequenceEncodingDecoder.java @@ -4,7 +4,7 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.core.proto.ProtoSequenceMetadata; import io.github.dfa1.vortex.reader.array.Array; @@ -72,9 +72,9 @@ private static Array decodeInteger( long v = base + i * mul; switch (pt) { case I8, U8 -> seg.set(ValueLayout.JAVA_BYTE, i, (byte) v); - case I16, U16 -> seg.setAtIndex(PTypeIO.LE_SHORT, i, (short) v); - case I32, U32 -> seg.setAtIndex(PTypeIO.LE_INT, i, (int) v); - case I64, U64 -> seg.setAtIndex(PTypeIO.LE_LONG, i, v); + case I16, U16 -> seg.setAtIndex(VortexFormat.LE_SHORT, i, (short) v); + case I32, U32 -> seg.setAtIndex(VortexFormat.LE_INT, i, (int) v); + case I64, U64 -> seg.setAtIndex(VortexFormat.LE_LONG, i, v); default -> throw new IllegalStateException("unreachable"); } } @@ -92,7 +92,7 @@ private static Array decodeF32(ProtoSequenceMetadata meta, long n, DType dtype, float mul = meta.multiplier().f32_value(); MemorySegment seg = arena.allocate(n * 4L); for (long i = 0; i < n; i++) { - seg.setAtIndex(PTypeIO.LE_FLOAT, i, base + i * mul); + seg.setAtIndex(VortexFormat.LE_FLOAT, i, base + i * mul); } return new MaterializedFloatArray(dtype, n, seg); } @@ -102,7 +102,7 @@ private static Array decodeF64(ProtoSequenceMetadata meta, long n, DType dtype, double mul = meta.multiplier().f64_value(); MemorySegment seg = arena.allocate(n * 8L); for (long i = 0; i < n; i++) { - seg.setAtIndex(PTypeIO.LE_DOUBLE, i, base + i * mul); + seg.setAtIndex(VortexFormat.LE_DOUBLE, i, base + i * mul); } return new MaterializedDoubleArray(dtype, n, seg); } @@ -114,7 +114,7 @@ private static Array decodeF16(ProtoSequenceMetadata meta, long n, DType dtype, float mul = Float.float16ToFloat(mulShort); MemorySegment seg = arena.allocate(n * 2L); for (long i = 0; i < n; i++) { - seg.setAtIndex(PTypeIO.LE_SHORT, i, Float.floatToFloat16(base + i * mul)); + seg.setAtIndex(VortexFormat.LE_SHORT, i, Float.floatToFloat16(base + i * mul)); } return new MaterializedFloat16Array(dtype, n, seg); } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoder.java index c530def2..1626ecfe 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/SparseEncodingDecoder.java @@ -4,7 +4,7 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoPatchesMetadata; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.core.proto.ProtoSparseMetadata; @@ -161,7 +161,7 @@ private static Array decodeVarBin( patchCursor++; } } - outOffsets.setAtIndex(PTypeIO.LE_INT, pos + 1, (int) bytePos); + outOffsets.setAtIndex(VortexFormat.LE_INT, pos + 1, (int) bytePos); } return new VarBinArray.OffsetMode(ctx.dtype(), n, outBytes, outOffsets, PType.I32); @@ -169,8 +169,8 @@ private static Array decodeVarBin( private static long readVarBinOffset(MemorySegment seg, long i, PType ptype) { return switch (ptype) { - case I32, U32 -> Integer.toUnsignedLong(seg.getAtIndex(PTypeIO.LE_INT, i)); - case I64, U64 -> seg.getAtIndex(PTypeIO.LE_LONG, i); + case I32, U32 -> Integer.toUnsignedLong(seg.getAtIndex(VortexFormat.LE_INT, i)); + case I64, U64 -> seg.getAtIndex(VortexFormat.LE_LONG, i); default -> throw new VortexException(EncodingId.VORTEX_SPARSE, "unsupported offset ptype " + ptype); }; } @@ -178,9 +178,9 @@ private static long readVarBinOffset(MemorySegment seg, long i, PType ptype) { private static long readUnsignedIdx(MemorySegment seg, long off, PType ptype) { return switch (ptype) { case U8 -> Byte.toUnsignedLong(seg.get(ValueLayout.JAVA_BYTE, off)); - case U16 -> Short.toUnsignedLong(seg.get(PTypeIO.LE_SHORT, off)); - case U32 -> Integer.toUnsignedLong(seg.get(PTypeIO.LE_INT, off)); - case U64 -> seg.get(PTypeIO.LE_LONG, off); + case U16 -> Short.toUnsignedLong(seg.get(VortexFormat.LE_SHORT, off)); + case U32 -> Integer.toUnsignedLong(seg.get(VortexFormat.LE_INT, off)); + case U64 -> seg.get(VortexFormat.LE_LONG, off); default -> throw new VortexException(EncodingId.VORTEX_SPARSE, "non-unsigned index ptype " + ptype); }; } diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoder.java index 2e6f2d61..c366a241 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZigZagEncodingDecoder.java @@ -4,7 +4,7 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.LazyConstantByteArray; import io.github.dfa1.vortex.reader.array.LazyConstantIntArray; @@ -51,15 +51,15 @@ public Array decode(DecodeContext ctx) { yield new LazyConstantByteArray(ctx.dtype(), n, (byte) ((u >>> 1) ^ -(u & 1))); } case I16 -> { - int u = Short.toUnsignedInt(src.get(PTypeIO.LE_SHORT, 0)); + int u = Short.toUnsignedInt(src.get(VortexFormat.LE_SHORT, 0)); yield new LazyConstantShortArray(ctx.dtype(), n, (short) ((u >>> 1) ^ -(u & 1))); } case I32 -> { - int u = src.get(PTypeIO.LE_INT, 0); + int u = src.get(VortexFormat.LE_INT, 0); yield new LazyConstantIntArray(ctx.dtype(), n, (u >>> 1) ^ -(u & 1)); } case I64 -> { - long u = src.get(PTypeIO.LE_LONG, 0); + long u = src.get(VortexFormat.LE_LONG, 0); yield new LazyConstantLongArray(ctx.dtype(), n, (u >>> 1) ^ -(u & 1L)); } default -> throw new VortexException(EncodingId.VORTEX_ZIGZAG, "unreachable"); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZstdEncodingDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZstdEncodingDecoder.java index 97227a8d..5f68d10e 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZstdEncodingDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/decode/ZstdEncodingDecoder.java @@ -4,8 +4,8 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.io.IoBounds; -import io.github.dfa1.vortex.core.io.PTypeIO; import io.github.dfa1.vortex.core.proto.ProtoZstdMetadata; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.BoolArray; @@ -131,7 +131,7 @@ private static VarBinArray buildScatteredVarBin( MemorySegment values = ctx.arena().allocate(totalDataBytes > 0 ? totalDataBytes : 1); MemorySegment offsets = ctx.arena().allocate((rowCount + 1) * 4L, 4); - offsets.setAtIndex(PTypeIO.LE_INT, 0, 0); + offsets.setAtIndex(VortexFormat.LE_INT, 0, 0); // Second pass reads the same positions the first pass already bounds-checked via // readVarBinLen, so a raw get/copy here cannot overrun; values is sized to the validated @@ -140,13 +140,13 @@ private static VarBinArray buildScatteredVarBin( long dataPos = 0; for (long i = 0; i < rowCount; i++) { if (validity.getBoolean(i)) { - int len = validValues.get(PTypeIO.LE_INT, readPos); + int len = validValues.get(VortexFormat.LE_INT, readPos); readPos += 4; MemorySegment.copy(validValues, readPos, values, dataPos, len); readPos += len; dataPos += len; } - offsets.setAtIndex(PTypeIO.LE_INT, i + 1, (int) dataPos); + offsets.setAtIndex(VortexFormat.LE_INT, i + 1, (int) dataPos); } return new VarBinArray.OffsetMode(dtype.withNullable(false), rowCount, values, offsets, PType.I32); @@ -261,7 +261,7 @@ private static Array buildPrimitive(DType.Primitive dt, long n, MemorySegment de /// @return the validated element length in bytes private static int readVarBinLen(MemorySegment src, long pos) { IoBounds.checkRange(pos, 4, src.byteSize()); - int len = src.get(PTypeIO.LE_INT, pos); + int len = src.get(VortexFormat.LE_INT, pos); // checkRange rejects len < 0 and a [pos+4, pos+4+len) range that overruns src. IoBounds.checkRange(pos + 4L, len, src.byteSize()); return len; @@ -278,7 +278,7 @@ private static VarBinArray buildVarBin(DType dtype, long n, MemorySegment decomp MemorySegment values = ctx.arena().allocate(totalDataBytes); MemorySegment offsets = ctx.arena().allocate((n + 1) * 4L, 4); - offsets.setAtIndex(PTypeIO.LE_INT, 0, 0); + offsets.setAtIndex(VortexFormat.LE_INT, 0, 0); // Second pass reads the same positions the first pass already bounds-checked via // readVarBinLen, so a raw get/copy here cannot overrun; values is sized to the validated @@ -286,12 +286,12 @@ private static VarBinArray buildVarBin(DType dtype, long n, MemorySegment decomp pos = 0; long dataPos = 0; for (long i = 0; i < n; i++) { - int len = decompressed.get(PTypeIO.LE_INT, pos); + int len = decompressed.get(VortexFormat.LE_INT, pos); pos += 4; MemorySegment.copy(decompressed, pos, values, dataPos, len); pos += len; dataPos += len; - offsets.setAtIndex(PTypeIO.LE_INT, i + 1, (int) dataPos); + offsets.setAtIndex(VortexFormat.LE_INT, i + 1, (int) dataPos); } return new VarBinArray.OffsetMode(dtype, n, values, offsets, PType.I32); diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java index ce5edee0..4c2608f7 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/DictLayoutDecoder.java @@ -1,8 +1,8 @@ package io.github.dfa1.vortex.reader.layout; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_SHORT; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_INT; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_LONG; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_SHORT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_LONG; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.DType; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchema.java b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchema.java index e592e31d..818dd0d4 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchema.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchema.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.core.model.DType; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_INT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayNodeDepthBombSecurityTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayNodeDepthBombSecurityTest.java index d39c5511..72fb2198 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayNodeDepthBombSecurityTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayNodeDepthBombSecurityTest.java @@ -4,7 +4,7 @@ import io.github.dfa1.vortex.core.fbs.FbsArray; import io.github.dfa1.vortex.core.fbs.FbsArrayNode; import io.github.dfa1.vortex.core.fbs.FbsBuilder; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.DType; import org.junit.jupiter.api.Test; @@ -66,7 +66,7 @@ void arrayTreeOneOverDepthLimit_throwsVortexException() { private static MemorySegment wrapAsSegment(byte[] fb, Arena arena) { MemorySegment seg = arena.allocate(fb.length + 4L); MemorySegment.copy(MemorySegment.ofArray(fb), 0, seg, 0, fb.length); - seg.set(PTypeIO.LE_INT, fb.length, fb.length); + seg.set(VortexFormat.LE_INT, fb.length, fb.length); return seg; } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ComputeFilteredAggregateTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ComputeFilteredAggregateTest.java index 959bff50..0a19b8bb 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/ComputeFilteredAggregateTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ComputeFilteredAggregateTest.java @@ -1,6 +1,7 @@ package io.github.dfa1.vortex.reader; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; +import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.BoolArray; @@ -33,6 +34,7 @@ import java.util.List; import java.util.Map; import java.util.Random; +import java.util.SequencedMap; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -271,9 +273,9 @@ void dictFilterUnsignedAggregateKeepsUnsignedOrder() { // comparison instead of the kernel's unsigned-aware order, 2^63 would come out as the min Array filter = dictColumn(new long[]{1, 2}, new int[]{1, 0, 1}); MemorySegment seg = ARENA.allocate(24, 8); - seg.setAtIndex(PTypeIO.LE_LONG, 0, Long.MIN_VALUE); - seg.setAtIndex(PTypeIO.LE_LONG, 1, 7L); - seg.setAtIndex(PTypeIO.LE_LONG, 2, 5L); + seg.setAtIndex(VortexFormat.LE_LONG, 0, Long.MIN_VALUE); + seg.setAtIndex(VortexFormat.LE_LONG, 1, 7L); + seg.setAtIndex(VortexFormat.LE_LONG, 2, 5L); Array agg = new MaterializedLongArray(DType.U64, 3, seg); Chunk chunk = chunk(3, Map.of("f0", filter, "v", agg)); @@ -566,7 +568,7 @@ private static Array dictArray(Reference reference, Random random) { } MemorySegment poolSeg = ARENA.allocate(Math.max(8L, pool.size() * 8L), 8); for (int i = 0; i < pool.size(); i++) { - poolSeg.setAtIndex(PTypeIO.LE_LONG, i, pool.get(i)); + poolSeg.setAtIndex(VortexFormat.LE_LONG, i, pool.get(i)); } MaterializedLongArray poolArray = new MaterializedLongArray(DType.I64, pool.size(), poolSeg); ByteArray codesArray; @@ -588,7 +590,7 @@ private static Array dictArray(Reference reference, Random random) { private static Array dictColumn(long[] pool, int[] codes) { MemorySegment poolSeg = ARENA.allocate(Math.max(8L, pool.length * 8L), 8); for (int i = 0; i < pool.length; i++) { - poolSeg.setAtIndex(PTypeIO.LE_LONG, i, pool[i]); + poolSeg.setAtIndex(VortexFormat.LE_LONG, i, pool[i]); } MaterializedLongArray poolArray = new MaterializedLongArray(DType.I64, pool.length, poolSeg); return DictLongArray.of(DType.I64, codes.length, poolArray, byteArray(codes, 0, codes.length)); @@ -598,7 +600,7 @@ private static Array dictColumn(long[] pool, int[] codes) { private static Array dictDoubleColumn(double[] pool, int[] codes) { MemorySegment poolSeg = ARENA.allocate(Math.max(8L, pool.length * 8L), 8); for (int i = 0; i < pool.length; i++) { - poolSeg.setAtIndex(PTypeIO.LE_DOUBLE, i, pool[i]); + poolSeg.setAtIndex(VortexFormat.LE_DOUBLE, i, pool[i]); } MaterializedDoubleArray poolArray = new MaterializedDoubleArray(DType.F64, pool.length, poolSeg); return DictDoubleArray.of(DType.F64, codes.length, poolArray, byteArray(codes, 0, codes.length)); @@ -615,15 +617,15 @@ private static MaterializedByteArray byteArray(int[] codes, int from, int to) { private static Array floatArray(float... values) { MemorySegment segment = ARENA.allocate(Math.max(4L, values.length * 4L), 4); for (int i = 0; i < values.length; i++) { - segment.setAtIndex(PTypeIO.LE_FLOAT, i, values[i]); + segment.setAtIndex(VortexFormat.LE_FLOAT, i, values[i]); } return new MaterializedFloatArray(DType.F32, values.length, segment); } private static Chunk chunk(long rows, Map columns) { - Map dtypes = new LinkedHashMap<>(); - columns.forEach((name, array) -> dtypes.put(name, array.dtype())); - return new Chunk(rows, columns, dtypes, ARENA, c -> { /* no-op: the shared arena outlives the chunk */ }); + SequencedMap typed = new LinkedHashMap<>(); + columns.forEach((name, array) -> typed.put(ColumnName.of(name), new Chunk.Column(array, array.dtype()))); + return new Chunk(rows, typed, ARENA, c -> { /* no-op: the shared arena outlives the chunk */ }); } /// Builds a long-domain column from a reference, optionally narrowing it to an `i32` array, and @@ -635,13 +637,13 @@ private static Array longArray(Reference reference, boolean nullable) { if (values.length % 2 == 0) { MemorySegment segment = ARENA.allocate(Math.max(8L, values.length * 8L), 8); for (int i = 0; i < values.length; i++) { - segment.setAtIndex(PTypeIO.LE_LONG, i, values[i]); + segment.setAtIndex(VortexFormat.LE_LONG, i, values[i]); } data = new MaterializedLongArray(DType.I64, values.length, segment); } else { MemorySegment segment = ARENA.allocate(Math.max(4L, values.length * 4L), 4); for (int i = 0; i < values.length; i++) { - segment.setAtIndex(PTypeIO.LE_INT, i, (int) values[i]); + segment.setAtIndex(VortexFormat.LE_INT, i, (int) values[i]); } data = new MaterializedIntArray(DType.I32, values.length, segment); } @@ -655,7 +657,7 @@ private static Array longArray(Reference reference, boolean nullable) { private static Array doubleArray(double[] values) { MemorySegment segment = ARENA.allocate(Math.max(8L, values.length * 8L), 8); for (int i = 0; i < values.length; i++) { - segment.setAtIndex(PTypeIO.LE_DOUBLE, i, values[i]); + segment.setAtIndex(VortexFormat.LE_DOUBLE, i, values[i]); } return new MaterializedDoubleArray(DType.F64, values.length, segment); } @@ -663,7 +665,7 @@ private static Array doubleArray(double[] values) { private static Array float16Array(float[] values) { MemorySegment segment = ARENA.allocate(Math.max(2L, values.length * 2L), 2); for (int i = 0; i < values.length; i++) { - segment.setAtIndex(PTypeIO.LE_SHORT, i, Float.floatToFloat16(values[i])); + segment.setAtIndex(VortexFormat.LE_SHORT, i, Float.floatToFloat16(values[i])); } return new MaterializedFloat16Array(DType.F16, values.length, segment); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/SerializedArrayBoundsSecurityTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/SerializedArrayBoundsSecurityTest.java index 803c48ca..6b947eca 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/SerializedArrayBoundsSecurityTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/SerializedArrayBoundsSecurityTest.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.reader; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.fbs.FbsBuilder; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.error.VortexException; @@ -46,7 +46,7 @@ void declaredFbLenLargerThanSegment_throwsVortexException() { // Given a 16-byte segment whose trailing u32 claims a 1 000 000-byte FlatBuffer; // fbStart = segLen - 4 - fbLen would go deeply negative. MemorySegment seg = arena.allocate(16); - seg.set(PTypeIO.LE_INT, 12, 1_000_000); + seg.set(VortexFormat.LE_INT, 12, 1_000_000); // When / Then assertThatThrownBy(() -> sut.decode(seg, List.of("vortex.flat"), DTYPE, 1, arena)) @@ -59,7 +59,7 @@ void negativeFbLen_throwsVortexException() { try (Arena arena = Arena.ofConfined()) { // Given a trailing u32 of 0xFFFFFFFF — reads back as a signed int of -1 MemorySegment seg = arena.allocate(16); - seg.set(PTypeIO.LE_INT, 12, -1); + seg.set(VortexFormat.LE_INT, 12, -1); // When / Then the negative length is rejected (checkRange len < 0) assertThatThrownBy(() -> sut.decode(seg, List.of("vortex.flat"), DTYPE, 1, arena)) @@ -75,7 +75,7 @@ void bufferDescriptorLengthPastSegment_throwsVortexException() { byte[] fb = arrayFlatBufferWithOneBuffer(1_000_000L); MemorySegment seg = arena.allocate(fb.length + 4L); MemorySegment.copy(MemorySegment.ofArray(fb), 0, seg, 0, fb.length); - seg.set(PTypeIO.LE_INT, fb.length, fb.length); + seg.set(VortexFormat.LE_INT, fb.length, fb.length); // When / Then the buffer slice is bounds-checked before asSlice assertThatThrownBy(() -> sut.decode(seg, List.of("vortex.flat"), DTYPE, 1, arena)) diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/SerializedArrayDecoderDecodeTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/SerializedArrayDecoderDecodeTest.java index b6b01cef..72b5aa5b 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/SerializedArrayDecoderDecodeTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/SerializedArrayDecoderDecodeTest.java @@ -13,7 +13,7 @@ import java.lang.foreign.MemorySegment; import java.util.List; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_INT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.catchThrowable; diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/TrailerLengthBoundaryTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/TrailerLengthBoundaryTest.java index a0502804..83efd062 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/TrailerLengthBoundaryTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/TrailerLengthBoundaryTest.java @@ -6,7 +6,7 @@ import java.lang.foreign.MemorySegment; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_SHORT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_SHORT; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/VortexReaderDecodeChunkTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/VortexReaderDecodeChunkTest.java index e3f4a758..da7ffc28 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/VortexReaderDecodeChunkTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/VortexReaderDecodeChunkTest.java @@ -1,6 +1,7 @@ package io.github.dfa1.vortex.reader; import io.github.dfa1.vortex.core.error.VortexException; +import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.fbs.FbsArray; import io.github.dfa1.vortex.core.fbs.FbsArrayNode; import io.github.dfa1.vortex.core.fbs.FbsBuilder; @@ -87,7 +88,7 @@ void decodeChunk_columnSubset_matchesStreaming(int chunkIndex, @TempDir Path tmp List resultA; try (Chunk result = reader.decodeChunk(chunkIndex, List.of("a"))) { // Then — only the projected column is present, and it matches the stream - assertThat(result.columns().keySet()).containsExactly("a"); + assertThat(result.columns().keySet()).containsExactly(ColumnName.of("a")); resultA = values(result.column("a")); } assertThat(resultA).isEqualTo(streamed.get(chunkIndex).get("a")); @@ -104,7 +105,8 @@ void decodeChunk_emptyColumns_decodesAllColumns(@TempDir Path tmp) throws Except try (Chunk result = reader.decodeChunk(0, List.of())) { // Then — both columns decoded - assertThat(result.columns().keySet()).containsExactlyInAnyOrder("a", "b"); + assertThat(result.columns().keySet()) + .containsExactlyInAnyOrder(ColumnName.of("a"), ColumnName.of("b")); } } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/VortexReaderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/VortexReaderTest.java index 7e25c922..4625b9d2 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/VortexReaderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/VortexReaderTest.java @@ -195,9 +195,9 @@ void scan_withNoDecoders_allowUnknown_returnsUnknownArray(String name) throws UR try (Chunk chunk = iter.next()) { assertThat(chunk.rowCount()).isGreaterThan(0); assertThat(chunk.columns()).isNotEmpty(); - for (Array column : chunk.columns().values()) { - assertThat(column).isInstanceOf(UnknownArray.class); - UnknownArray foreign = (UnknownArray) column; + for (Chunk.Column column : chunk.columns().values()) { + assertThat(column.array()).isInstanceOf(UnknownArray.class); + UnknownArray foreign = (UnknownArray) column.array(); assertThat(foreign.encodingId().id()).describedAs(name).startsWith("vortex."); } } @@ -265,8 +265,8 @@ void scan_withLimit_returnsExactlyNRows() throws URISyntaxException, IOException while (iter.hasNext()) { try (Chunk chunk = iter.next()) { totalRows += chunk.rowCount(); - for (Array col : chunk.columns().values()) { - assertThat(col.length()).isLessThanOrEqualTo(limit); + for (Chunk.Column col : chunk.columns().values()) { + assertThat(col.array().length()).isLessThanOrEqualTo(limit); } } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/ArrayMaterializeTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/ArrayMaterializeTest.java index 6d00d848..41441c0f 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/ArrayMaterializeTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/ArrayMaterializeTest.java @@ -3,7 +3,7 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.error.VortexException; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -76,8 +76,8 @@ void longViewDecodesEveryElementThroughGetLong() { // Then values come back little-endian in order assertThat(result.byteSize()).isEqualTo(3 * 8L); - assertThat(result.getAtIndex(PTypeIO.LE_LONG, 0)).isEqualTo(10L); - assertThat(result.getAtIndex(PTypeIO.LE_LONG, 2)).isEqualTo(30L); + assertThat(result.getAtIndex(VortexFormat.LE_LONG, 0)).isEqualTo(10L); + assertThat(result.getAtIndex(VortexFormat.LE_LONG, 2)).isEqualTo(30L); } @Test @@ -111,8 +111,8 @@ void frameOfReferenceAddsReference() { MemorySegment result = sut.materialize(arena); // Then each element is decoded + ref - assertThat(result.getAtIndex(PTypeIO.LE_LONG, 0)).isEqualTo(101L); - assertThat(result.getAtIndex(PTypeIO.LE_LONG, 2)).isEqualTo(103L); + assertThat(result.getAtIndex(VortexFormat.LE_LONG, 0)).isEqualTo(101L); + assertThat(result.getAtIndex(VortexFormat.LE_LONG, 2)).isEqualTo(103L); } @Test @@ -124,10 +124,10 @@ void zigzagDecodesSignedZigzagPattern() { MemorySegment result = sut.materialize(arena); // Then - assertThat(result.getAtIndex(PTypeIO.LE_LONG, 0)).isEqualTo(0L); - assertThat(result.getAtIndex(PTypeIO.LE_LONG, 1)).isEqualTo(-1L); - assertThat(result.getAtIndex(PTypeIO.LE_LONG, 2)).isEqualTo(1L); - assertThat(result.getAtIndex(PTypeIO.LE_LONG, 3)).isEqualTo(-2L); + assertThat(result.getAtIndex(VortexFormat.LE_LONG, 0)).isEqualTo(0L); + assertThat(result.getAtIndex(VortexFormat.LE_LONG, 1)).isEqualTo(-1L); + assertThat(result.getAtIndex(VortexFormat.LE_LONG, 2)).isEqualTo(1L); + assertThat(result.getAtIndex(VortexFormat.LE_LONG, 3)).isEqualTo(-2L); } @Test @@ -139,8 +139,8 @@ void alpAppliesBothFactors() { MemorySegment result = sut.materialize(arena); // Then - assertThat(result.getAtIndex(PTypeIO.LE_DOUBLE, 0)).isEqualTo(1.0); - assertThat(result.getAtIndex(PTypeIO.LE_DOUBLE, 2)).isEqualTo(3.0); + assertThat(result.getAtIndex(VortexFormat.LE_DOUBLE, 0)).isEqualTo(1.0); + assertThat(result.getAtIndex(VortexFormat.LE_DOUBLE, 2)).isEqualTo(3.0); } } @@ -158,9 +158,9 @@ void chunkedConcatenatesChildrenInOrder() { // Then one contiguous segment spanning both chunks assertThat(result.byteSize()).isEqualTo(5 * 8L); - assertThat(result.getAtIndex(PTypeIO.LE_LONG, 0)).isEqualTo(0L); - assertThat(result.getAtIndex(PTypeIO.LE_LONG, 3)).isEqualTo(3L); - assertThat(result.getAtIndex(PTypeIO.LE_LONG, 4)).isEqualTo(4L); + assertThat(result.getAtIndex(VortexFormat.LE_LONG, 0)).isEqualTo(0L); + assertThat(result.getAtIndex(VortexFormat.LE_LONG, 3)).isEqualTo(3L); + assertThat(result.getAtIndex(VortexFormat.LE_LONG, 4)).isEqualTo(4L); } @Test @@ -173,9 +173,9 @@ void dictGathersOneValuePerCode() { // Then each row resolves to values[code] assertThat(result.byteSize()).isEqualTo(3 * 8L); - assertThat(result.getAtIndex(PTypeIO.LE_LONG, 0)).isEqualTo(10L); - assertThat(result.getAtIndex(PTypeIO.LE_LONG, 1)).isEqualTo(20L); - assertThat(result.getAtIndex(PTypeIO.LE_LONG, 2)).isEqualTo(10L); + assertThat(result.getAtIndex(VortexFormat.LE_LONG, 0)).isEqualTo(10L); + assertThat(result.getAtIndex(VortexFormat.LE_LONG, 1)).isEqualTo(20L); + assertThat(result.getAtIndex(VortexFormat.LE_LONG, 2)).isEqualTo(10L); } } @@ -194,8 +194,8 @@ void constantDecimalFillsValueEveryRow() { // Then every row holds the same little-endian mantissa assertThat(result.byteSize()).isEqualTo(3 * 8L); - assertThat(result.getAtIndex(PTypeIO.LE_LONG, 0)).isEqualTo(12345L); - assertThat(result.getAtIndex(PTypeIO.LE_LONG, 2)).isEqualTo(12345L); + assertThat(result.getAtIndex(VortexFormat.LE_LONG, 0)).isEqualTo(12345L); + assertThat(result.getAtIndex(VortexFormat.LE_LONG, 2)).isEqualTo(12345L); } } @@ -211,8 +211,8 @@ void delegatesToInnerDataIgnoringMask() { MemorySegment result = sut.materialize(arena); // Then the inner data segment is returned (validity is not surfaced here) - assertThat(result.getAtIndex(PTypeIO.LE_LONG, 0)).isEqualTo(7L); - assertThat(result.getAtIndex(PTypeIO.LE_LONG, 2)).isEqualTo(9L); + assertThat(result.getAtIndex(VortexFormat.LE_LONG, 0)).isEqualTo(7L); + assertThat(result.getAtIndex(VortexFormat.LE_LONG, 2)).isEqualTo(9L); } } @@ -331,8 +331,8 @@ void floatViewDecodesEveryElementThroughGetFloat() { // Then the kept prefix is written as little-endian f32 assertThat(result.byteSize()).isEqualTo(3 * 4L); - assertThat(result.getAtIndex(PTypeIO.LE_FLOAT, 0)).isEqualTo(1.5f); - assertThat(result.getAtIndex(PTypeIO.LE_FLOAT, 2)).isEqualTo(3.5f); + assertThat(result.getAtIndex(VortexFormat.LE_FLOAT, 0)).isEqualTo(1.5f); + assertThat(result.getAtIndex(VortexFormat.LE_FLOAT, 2)).isEqualTo(3.5f); } @Test @@ -359,8 +359,8 @@ void shortViewDecodesEveryElementThroughGetShort() { // Then little-endian i16 assertThat(result.byteSize()).isEqualTo(3 * 2L); - assertThat(result.getAtIndex(PTypeIO.LE_SHORT, 0)).isEqualTo((short) 100); - assertThat(result.getAtIndex(PTypeIO.LE_SHORT, 2)).isEqualTo((short) 300); + assertThat(result.getAtIndex(VortexFormat.LE_SHORT, 0)).isEqualTo((short) 100); + assertThat(result.getAtIndex(VortexFormat.LE_SHORT, 2)).isEqualTo((short) 300); } @Test @@ -373,8 +373,8 @@ void intViewDecodesEveryElementThroughGetInt() { // Then little-endian i32 assertThat(result.byteSize()).isEqualTo(3 * 4L); - assertThat(result.getAtIndex(PTypeIO.LE_INT, 0)).isEqualTo(1); - assertThat(result.getAtIndex(PTypeIO.LE_INT, 2)).isEqualTo(3); + assertThat(result.getAtIndex(VortexFormat.LE_INT, 0)).isEqualTo(1); + assertThat(result.getAtIndex(VortexFormat.LE_INT, 2)).isEqualTo(3); } @Test @@ -387,15 +387,15 @@ void doubleViewDecodesEveryElementThroughGetDouble() { // Then little-endian f64 assertThat(result.byteSize()).isEqualTo(2 * 8L); - assertThat(result.getAtIndex(PTypeIO.LE_DOUBLE, 0)).isEqualTo(1.5); - assertThat(result.getAtIndex(PTypeIO.LE_DOUBLE, 1)).isEqualTo(2.5); + assertThat(result.getAtIndex(VortexFormat.LE_DOUBLE, 0)).isEqualTo(1.5); + assertThat(result.getAtIndex(VortexFormat.LE_DOUBLE, 1)).isEqualTo(2.5); } } private MemorySegment encodedLongs(long... vs) { MemorySegment seg = arena.allocate(vs.length * 8L, 8); for (int i = 0; i < vs.length; i++) { - seg.setAtIndex(PTypeIO.LE_LONG, i, vs[i]); + seg.setAtIndex(VortexFormat.LE_LONG, i, vs[i]); } return seg; } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/ChunkedRecordSmokeTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/ChunkedRecordSmokeTest.java index 5d3420b5..348d4102 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/ChunkedRecordSmokeTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/ChunkedRecordSmokeTest.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.reader.array; import io.github.dfa1.vortex.core.error.VortexException; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -380,7 +380,7 @@ void forEachFoldAndMaterialize() { try (Arena arena = Arena.ofConfined()) { MemorySegment m = sut.materialize(arena); for (int i = 0; i < 4; i++) { - assertThat(m.getAtIndex(PTypeIO.LE_INT, i)).isEqualTo(sut.getInt(i)); + assertThat(m.getAtIndex(VortexFormat.LE_INT, i)).isEqualTo(sut.getInt(i)); } } } @@ -451,7 +451,7 @@ void forEachFoldAndMaterialize() { try (Arena arena = Arena.ofConfined()) { MemorySegment m = sut.materialize(arena); for (int i = 0; i < 3; i++) { - assertThat(m.getAtIndex(PTypeIO.LE_DOUBLE, i)).isEqualTo(sut.getDouble(i)); + assertThat(m.getAtIndex(VortexFormat.LE_DOUBLE, i)).isEqualTo(sut.getDouble(i)); } } } @@ -503,7 +503,7 @@ void foldAndMaterialize() { try (Arena arena = Arena.ofConfined()) { MemorySegment m = sut.materialize(arena); for (int i = 0; i < 3; i++) { - assertThat(m.getAtIndex(PTypeIO.LE_FLOAT, i)).isEqualTo(sut.getFloat(i)); + assertThat(m.getAtIndex(VortexFormat.LE_FLOAT, i)).isEqualTo(sut.getFloat(i)); } } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/DictRecordSmokeTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/DictRecordSmokeTest.java index 5164ff5d..70561fc1 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/DictRecordSmokeTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/DictRecordSmokeTest.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.error.VortexException; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -186,7 +186,7 @@ void allCodeTypes_getForEachFoldMaterialize() { // materialize MemorySegment m = sut.materialize(arena); for (int i = 0; i < 3; i++) { - assertThat(m.getAtIndex(PTypeIO.LE_LONG, i)).as(label).isEqualTo(expected[i]); + assertThat(m.getAtIndex(VortexFormat.LE_LONG, i)).as(label).isEqualTo(expected[i]); } } } @@ -276,7 +276,7 @@ void allCodeTypes_getForEachFoldMaterialize() { assertThat(sut.fold(0, Integer::sum)).as(label).isEqualTo(600); MemorySegment m = sut.materialize(arena); for (int i = 0; i < 3; i++) { - assertThat(m.getAtIndex(PTypeIO.LE_INT, i)).as(label).isEqualTo(expected[i]); + assertThat(m.getAtIndex(VortexFormat.LE_INT, i)).as(label).isEqualTo(expected[i]); } } } @@ -364,7 +364,7 @@ void allCodeTypes_getForEachFoldMaterialize() { assertThat(sut.fold(0.0, Double::sum)).as(label).isEqualTo(7.5); MemorySegment m = sut.materialize(arena); for (int i = 0; i < 3; i++) { - assertThat(m.getAtIndex(PTypeIO.LE_DOUBLE, i)).as(label).isEqualTo(expected[i]); + assertThat(m.getAtIndex(VortexFormat.LE_DOUBLE, i)).as(label).isEqualTo(expected[i]); } } } @@ -435,7 +435,7 @@ void allCodeTypes_getFoldMaterialize() { assertThat(sut.fold(0.0, Double::sum)).as(label).isEqualTo(7.5); MemorySegment m = sut.materialize(arena); for (int i = 0; i < 3; i++) { - assertThat(m.getAtIndex(PTypeIO.LE_FLOAT, i)).as(label).isEqualTo(expected[i]); + assertThat(m.getAtIndex(VortexFormat.LE_FLOAT, i)).as(label).isEqualTo(expected[i]); } } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/DoubleArrayTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/DoubleArrayTest.java index 166bed65..0e8b0581 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/DoubleArrayTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/DoubleArrayTest.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.reader.array; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.DType; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -66,7 +66,7 @@ void singleElement_consumerCalledOnce() { void logicalLengthExceedsCapacity_wrapsAround() { // Given — constant-encoding: 1-element buffer, logical length 3; all 3 visits yield same value MemorySegment seg = Arena.ofAuto().allocate(8, 8); - seg.setAtIndex(PTypeIO.LE_DOUBLE, 0, 2.71); + seg.setAtIndex(VortexFormat.LE_DOUBLE, 0, 2.71); DoubleArray sut = new MaterializedDoubleArray(DType.F64, 3, seg); List collected = new ArrayList<>(); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/IntArrayTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/IntArrayTest.java index f9c27e49..ad88963b 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/IntArrayTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/IntArrayTest.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.reader.array; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.DType; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -66,7 +66,7 @@ void singleElement_consumerCalledOnce() { void logicalLengthExceedsCapacity_wrapsAround() { // Given — constant-encoding: 1-element buffer, logical length 4; all 4 visits yield same value MemorySegment seg = Arena.ofAuto().allocate(4, 4); - seg.setAtIndex(PTypeIO.LE_INT, 0, 7); + seg.setAtIndex(VortexFormat.LE_INT, 0, 7); IntArray sut = new MaterializedIntArray(DType.I32, 4, seg); List collected = new ArrayList<>(); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyAlpDoubleArrayTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyAlpDoubleArrayTest.java index fc90ef9d..2342821a 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyAlpDoubleArrayTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyAlpDoubleArrayTest.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.reader.array; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.DType; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -21,7 +21,7 @@ class LazyAlpDoubleArrayTest { private static LazyAlpDoubleArray of(double scale, long... encoded) { MemorySegment seg = Arena.ofAuto().allocate((long) encoded.length * 8, 8); for (int i = 0; i < encoded.length; i++) { - seg.setAtIndex(PTypeIO.LE_LONG, i, encoded[i]); + seg.setAtIndex(VortexFormat.LE_LONG, i, encoded[i]); } return new LazyAlpDoubleArray(F64, encoded.length, seg, scale, 1.0); } @@ -122,7 +122,7 @@ void decodesAllRows() { // Then — each materialized row matches the lazy getter for (int i = 0; i < 3; i++) { - assertThat(seg.getAtIndex(PTypeIO.LE_DOUBLE, i)).as("row %d", i) + assertThat(seg.getAtIndex(VortexFormat.LE_DOUBLE, i)).as("row %d", i) .isCloseTo(sut.getDouble(i), offset(1e-12)); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyAlpFloatArrayTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyAlpFloatArrayTest.java index 043c9546..94f5aa53 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyAlpFloatArrayTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyAlpFloatArrayTest.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.reader.array; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.DType; import org.junit.jupiter.api.Test; @@ -18,7 +18,7 @@ class LazyAlpFloatArrayTest { private static LazyAlpFloatArray of(float scale, int... encoded) { MemorySegment seg = Arena.ofAuto().allocate((long) encoded.length * 4, 4); for (int i = 0; i < encoded.length; i++) { - seg.setAtIndex(PTypeIO.LE_INT, i, encoded[i]); + seg.setAtIndex(VortexFormat.LE_INT, i, encoded[i]); } return new LazyAlpFloatArray(F32, encoded.length, seg, scale, 1.0f); } @@ -57,7 +57,7 @@ void materializeDecodesAllRows() { // Then — each materialized row matches the lazy getter for (int i = 0; i < 3; i++) { - assertThat(seg.getAtIndex(PTypeIO.LE_FLOAT, i)).as("row %d", i) + assertThat(seg.getAtIndex(VortexFormat.LE_FLOAT, i)).as("row %d", i) .isCloseTo(sut.getFloat(i), offset(1e-6f)); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyConstantArrayTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyConstantArrayTest.java index 847be99d..ae68ad0f 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyConstantArrayTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyConstantArrayTest.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.error.VortexException; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -357,17 +357,17 @@ private int[] readByte(Arena arena) { private short[] readShort(Arena arena) { MemorySegment seg = new LazyConstantDecimalArray(dtype, 2, value, 2).materialize(arena); - return new short[]{seg.get(PTypeIO.LE_SHORT, 0), seg.get(PTypeIO.LE_SHORT, 2)}; + return new short[]{seg.get(VortexFormat.LE_SHORT, 0), seg.get(VortexFormat.LE_SHORT, 2)}; } private int[] readInt(Arena arena) { MemorySegment seg = new LazyConstantDecimalArray(dtype, 2, value, 4).materialize(arena); - return new int[]{seg.get(PTypeIO.LE_INT, 0), seg.get(PTypeIO.LE_INT, 4)}; + return new int[]{seg.get(VortexFormat.LE_INT, 0), seg.get(VortexFormat.LE_INT, 4)}; } private long[] readLong(Arena arena) { MemorySegment seg = new LazyConstantDecimalArray(dtype, 2, value, 8).materialize(arena); - return new long[]{seg.get(PTypeIO.LE_LONG, 0), seg.get(PTypeIO.LE_LONG, 8)}; + return new long[]{seg.get(VortexFormat.LE_LONG, 0), seg.get(VortexFormat.LE_LONG, 8)}; } } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyDecimalArrayTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyDecimalArrayTest.java index 6bfe53b4..324fb9c5 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyDecimalArrayTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyDecimalArrayTest.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.reader.array; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.error.VortexException; import org.junit.jupiter.api.Test; @@ -23,9 +23,9 @@ void getDecimal_i64Buffer_appliesDtypeScale() { // decimal(15, 2) with mantissa 4321 represents 43.21. try (Arena arena = Arena.ofConfined()) { MemorySegment buf = arena.allocate(24); - buf.set(PTypeIO.LE_LONG, 0, 4321L); - buf.set(PTypeIO.LE_LONG, 8, -100L); - buf.set(PTypeIO.LE_LONG, 16, 0L); + buf.set(VortexFormat.LE_LONG, 0, 4321L); + buf.set(VortexFormat.LE_LONG, 8, -100L); + buf.set(VortexFormat.LE_LONG, 16, 0L); DType.Decimal dec = new DType.Decimal((byte) 15, (byte) 2, false); LazyDecimalArray sut = new LazyDecimalArray(dec, 3, buf, 8); @@ -52,16 +52,16 @@ void getDecimal_i8I16I32Widths_signExtend() { assertThat(sut1.getDecimal(1)).isEqualByComparingTo(new BigDecimal("-0.5")); MemorySegment buf2 = arena.allocate(4); - buf2.set(PTypeIO.LE_SHORT, 0, (short) 1234); - buf2.set(PTypeIO.LE_SHORT, 2, (short) -7); + buf2.set(VortexFormat.LE_SHORT, 0, (short) 1234); + buf2.set(VortexFormat.LE_SHORT, 2, (short) -7); DType.Decimal dec2 = new DType.Decimal((byte) 4, (byte) 2, false); LazyDecimalArray sut2 = new LazyDecimalArray(dec2, 2, buf2, 2); assertThat(sut2.getDecimal(0)).isEqualByComparingTo(new BigDecimal("12.34")); assertThat(sut2.getDecimal(1)).isEqualByComparingTo(new BigDecimal("-0.07")); MemorySegment buf4 = arena.allocate(8); - buf4.set(PTypeIO.LE_INT, 0, 999_999); - buf4.set(PTypeIO.LE_INT, 4, -1); + buf4.set(VortexFormat.LE_INT, 0, 999_999); + buf4.set(VortexFormat.LE_INT, 4, -1); DType.Decimal dec4 = new DType.Decimal((byte) 9, (byte) 3, false); LazyDecimalArray sut4 = new LazyDecimalArray(dec4, 2, buf4, 4); assertThat(sut4.getDecimal(0)).isEqualByComparingTo(new BigDecimal("999.999")); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyForIntArrayTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyForIntArrayTest.java index 17a6679d..f7fd1d6d 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyForIntArrayTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyForIntArrayTest.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.reader.array; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.DType; import org.junit.jupiter.api.Test; @@ -19,7 +19,7 @@ class LazyForIntArrayTest { private static LazyForIntArray of(int ref, int... encoded) { MemorySegment seg = Arena.ofAuto().allocate((long) encoded.length * 4, 4); for (int i = 0; i < encoded.length; i++) { - seg.setAtIndex(PTypeIO.LE_INT, i, encoded[i]); + seg.setAtIndex(VortexFormat.LE_INT, i, encoded[i]); } return new LazyForIntArray(I32, encoded.length, seg, ref); } @@ -71,7 +71,7 @@ void materializeDecodesAllRows() { // Then — materialized rows match the lazy getter for (int i = 0; i < 3; i++) { - assertThat(seg.getAtIndex(PTypeIO.LE_INT, i)).as("row %d", i).isEqualTo(sut.getInt(i)); + assertThat(seg.getAtIndex(VortexFormat.LE_INT, i)).as("row %d", i).isEqualTo(sut.getInt(i)); } } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyForLongArrayTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyForLongArrayTest.java index 1b38d2f3..9256b285 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyForLongArrayTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyForLongArrayTest.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.reader.array; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.DType; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -20,7 +20,7 @@ class LazyForLongArrayTest { private static LazyForLongArray of(long ref, long... encoded) { MemorySegment seg = Arena.ofAuto().allocate((long) encoded.length * 8, 8); for (int i = 0; i < encoded.length; i++) { - seg.setAtIndex(PTypeIO.LE_LONG, i, encoded[i]); + seg.setAtIndex(VortexFormat.LE_LONG, i, encoded[i]); } return new LazyForLongArray(I64, encoded.length, seg, ref); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyForShortArrayTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyForShortArrayTest.java index 6194d3db..9ea82719 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyForShortArrayTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyForShortArrayTest.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.reader.array; import io.github.dfa1.vortex.core.model.DType; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import org.junit.jupiter.api.Test; import java.lang.foreign.Arena; @@ -18,7 +18,7 @@ class LazyForShortArrayTest { private static LazyForShortArray of(short ref, short... encoded) { MemorySegment seg = Arena.ofAuto().allocate((long) encoded.length * 2, 2); for (int i = 0; i < encoded.length; i++) { - seg.setAtIndex(PTypeIO.LE_SHORT, i, encoded[i]); + seg.setAtIndex(VortexFormat.LE_SHORT, i, encoded[i]); } return new LazyForShortArray(I16, encoded.length, seg, ref); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyZigZagIntArrayTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyZigZagIntArrayTest.java index 7dd692c2..0931fd72 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyZigZagIntArrayTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyZigZagIntArrayTest.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.reader.array; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.DType; import org.junit.jupiter.api.Test; @@ -19,7 +19,7 @@ class LazyZigZagIntArrayTest { private static LazyZigZagIntArray of(int... encoded) { MemorySegment seg = Arena.ofAuto().allocate((long) encoded.length * 4, 4); for (int i = 0; i < encoded.length; i++) { - seg.setAtIndex(PTypeIO.LE_INT, i, encoded[i]); + seg.setAtIndex(VortexFormat.LE_INT, i, encoded[i]); } return new LazyZigZagIntArray(I32, encoded.length, seg); } @@ -73,7 +73,7 @@ void materializeDecodesAllRows() { // Then — materialized rows match the lazy getter for (int i = 0; i < 5; i++) { - assertThat(seg.getAtIndex(PTypeIO.LE_INT, i)).as("row %d", i).isEqualTo(sut.getInt(i)); + assertThat(seg.getAtIndex(VortexFormat.LE_INT, i)).as("row %d", i).isEqualTo(sut.getInt(i)); } } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyZigZagLongArrayTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyZigZagLongArrayTest.java index 730e5d56..fead6802 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyZigZagLongArrayTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyZigZagLongArrayTest.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.reader.array; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.DType; import org.junit.jupiter.api.Test; @@ -19,7 +19,7 @@ class LazyZigZagLongArrayTest { private static LazyZigZagLongArray of(long... encoded) { MemorySegment seg = Arena.ofAuto().allocate((long) encoded.length * 8, 8); for (int i = 0; i < encoded.length; i++) { - seg.setAtIndex(PTypeIO.LE_LONG, i, encoded[i]); + seg.setAtIndex(VortexFormat.LE_LONG, i, encoded[i]); } return new LazyZigZagLongArray(I64, encoded.length, seg); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyZigZagShortArrayTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyZigZagShortArrayTest.java index 5a76b4a6..b6be7eda 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyZigZagShortArrayTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyZigZagShortArrayTest.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.reader.array; import io.github.dfa1.vortex.core.model.DType; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import org.junit.jupiter.api.Test; import java.lang.foreign.Arena; @@ -18,7 +18,7 @@ class LazyZigZagShortArrayTest { private static LazyZigZagShortArray of(short... encoded) { MemorySegment seg = Arena.ofAuto().allocate((long) encoded.length * 2, 2); for (int i = 0; i < encoded.length; i++) { - seg.setAtIndex(PTypeIO.LE_SHORT, i, encoded[i]); + seg.setAtIndex(VortexFormat.LE_SHORT, i, encoded[i]); } return new LazyZigZagShortArray(I16, encoded.length, seg); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LongArrayTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LongArrayTest.java index de3bd89c..a3ca38fe 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/LongArrayTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/LongArrayTest.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.reader.array; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.DType; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -66,7 +66,7 @@ void singleElement_consumerCalledOnce() { void logicalLengthExceedsCapacity_wrapsAround() { // Given — constant-encoding: 1-element buffer, logical length 4; all 4 visits yield same value MemorySegment seg = Arena.ofAuto().allocate(8, 8); - seg.setAtIndex(PTypeIO.LE_LONG, 0, 42L); + seg.setAtIndex(VortexFormat.LE_LONG, 0, 42L); LongArray sut = new MaterializedLongArray(DType.I64, 4, seg); List collected = new ArrayList<>(); diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinViewModeTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinViewModeTest.java index 8593eee2..1754e282 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinViewModeTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinViewModeTest.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.reader.array; import io.github.dfa1.vortex.core.model.DType; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -67,10 +67,10 @@ void getStringFromReferencedDataBuffer() { // One view referencing the data buffer at offset 0. MemorySegment views = arena.allocate(VIEW_SIZE); - views.set(PTypeIO.LE_INT, 0, longStr.length()); // size - views.set(PTypeIO.LE_INT, 4, 0); // prefix (ignored on read) - views.set(PTypeIO.LE_INT, 8, 0); // buffer index - views.set(PTypeIO.LE_INT, 12, 0); // offset within buffer + views.set(VortexFormat.LE_INT, 0, longStr.length()); // size + views.set(VortexFormat.LE_INT, 4, 0); // prefix (ignored on read) + views.set(VortexFormat.LE_INT, 8, 0); // buffer index + views.set(VortexFormat.LE_INT, 12, 0); // offset within buffer var sut = new VarBinArray.ViewMode(UTF8, 1, views, new MemorySegment[]{dataBuf}); // When/Then @@ -91,9 +91,9 @@ void mixedInlineAndReferenced() { MemorySegment views = arena.allocate(2L * VIEW_SIZE); writeInlineView(views, 0, "short"); // Row 1: referenced. - views.set(PTypeIO.LE_INT, VIEW_SIZE, longStr.length()); - views.set(PTypeIO.LE_INT, VIEW_SIZE + 8, 0); - views.set(PTypeIO.LE_INT, VIEW_SIZE + 12, 0); + views.set(VortexFormat.LE_INT, VIEW_SIZE, longStr.length()); + views.set(VortexFormat.LE_INT, VIEW_SIZE + 8, 0); + views.set(VortexFormat.LE_INT, VIEW_SIZE + 12, 0); var sut = new VarBinArray.ViewMode(UTF8, 2, views, new MemorySegment[]{dataBuf}); @@ -144,7 +144,7 @@ private static void writeInlineView(MemorySegment views, long viewOff, String s) if (bytes.length > 12) { throw new IllegalArgumentException("test helper only handles inline ≤ 12 byte values"); } - views.set(PTypeIO.LE_INT, viewOff, bytes.length); + views.set(VortexFormat.LE_INT, viewOff, bytes.length); for (int j = 0; j < bytes.length; j++) { views.set(ValueLayout.JAVA_BYTE, viewOff + 4 + j, bytes[j]); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ComputeArrays.java b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ComputeArrays.java index 7a2a3d9c..a9c144fa 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ComputeArrays.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ComputeArrays.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.reader.compute; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.reader.array.ByteArray; @@ -53,7 +53,7 @@ static MaterializedLongArray unsignedLongArray(Arena arena, long... values) { static MaterializedDoubleArray doubleArray(Arena arena, double... values) { MemorySegment seg = arena.allocate(Math.max(1, values.length * 8L), 8); for (int i = 0; i < values.length; i++) { - seg.setAtIndex(PTypeIO.LE_DOUBLE, i, values[i]); + seg.setAtIndex(VortexFormat.LE_DOUBLE, i, values[i]); } return new MaterializedDoubleArray(DType.F64, values.length, seg); } @@ -127,10 +127,10 @@ static VarBinArray utf8Array(Arena arena, String... values) { MemorySegment.copy(MemorySegment.ofArray(allBytes), 0, bytes, 0, allBytes.length); MemorySegment offsets = arena.allocate((values.length + 1) * Integer.BYTES, Integer.BYTES); int running = 0; - offsets.setAtIndex(PTypeIO.LE_INT, 0, 0); + offsets.setAtIndex(VortexFormat.LE_INT, 0, 0); for (int i = 0; i < values.length; i++) { running += values[i].getBytes(StandardCharsets.UTF_8).length; - offsets.setAtIndex(PTypeIO.LE_INT, i + 1, running); + offsets.setAtIndex(VortexFormat.LE_INT, i + 1, running); } return new VarBinArray.OffsetMode(DType.UTF8, values.length, bytes.asReadOnly(), offsets, PType.I32); @@ -139,7 +139,7 @@ static VarBinArray utf8Array(Arena arena, String... values) { private static MemorySegment longSegment(Arena arena, long... values) { MemorySegment seg = arena.allocate(Math.max(1, values.length * 8L), 8); for (int i = 0; i < values.length; i++) { - seg.setAtIndex(PTypeIO.LE_LONG, i, values[i]); + seg.setAtIndex(VortexFormat.LE_LONG, i, values[i]); } return seg; } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/DictFilterTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/DictFilterTest.java index 1a2661fb..8e7e0a67 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/DictFilterTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/DictFilterTest.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.reader.compute; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.ByteArray; @@ -187,7 +187,7 @@ void wideCodesDeclineTheLaneButStayCorrect() { MemorySegment seg = ARENA.allocate(10, 2); int[] codes = {0, 1, 2, 1, 0}; for (int i = 0; i < codes.length; i++) { - seg.setAtIndex(PTypeIO.LE_SHORT, i, (short) codes[i]); + seg.setAtIndex(VortexFormat.LE_SHORT, i, (short) codes[i]); } MaterializedShortArray wide = new MaterializedShortArray(DType.U16, codes.length, seg); Array filter = DictLongArray.of(DType.I64, codes.length, longPool(new long[]{7, 8, 9}), wide); @@ -306,7 +306,7 @@ private static Array dictOfLongs(Random random, DType.Primitive dtype, int poolS } MemorySegment seg = ARENA.allocate(Math.max(8L, poolSize * 8L), 8); for (int i = 0; i < poolSize; i++) { - seg.setAtIndex(PTypeIO.LE_LONG, i, pool[i]); + seg.setAtIndex(VortexFormat.LE_LONG, i, pool[i]); } MaterializedLongArray values = new MaterializedLongArray(dtype, poolSize, seg); ByteArray codeArray = randomCodesStructure(random, codes); @@ -322,7 +322,7 @@ private static Array dictOfLongs(Random random, DType.Primitive dtype, int poolS private static Array dictOfInts(Random random, int poolSize, int[] codes) { MemorySegment seg = ARENA.allocate(Math.max(4L, poolSize * 4L), 4); for (int i = 0; i < poolSize; i++) { - seg.setAtIndex(PTypeIO.LE_INT, i, random.nextInt(101) - 50); + seg.setAtIndex(VortexFormat.LE_INT, i, random.nextInt(101) - 50); } MaterializedIntArray values = new MaterializedIntArray(DType.I32, poolSize, seg); return DictIntArray.of(DType.I32, codes.length, values, randomCodesStructure(random, codes)); @@ -331,7 +331,7 @@ private static Array dictOfInts(Random random, int poolSize, int[] codes) { private static Array dictOfDoubles(Random random, int poolSize, int[] codes) { MemorySegment seg = ARENA.allocate(Math.max(8L, poolSize * 8L), 8); for (int i = 0; i < poolSize; i++) { - seg.setAtIndex(PTypeIO.LE_DOUBLE, i, random.nextInt(101) - 50); + seg.setAtIndex(VortexFormat.LE_DOUBLE, i, random.nextInt(101) - 50); } MaterializedDoubleArray values = new MaterializedDoubleArray(DType.F64, poolSize, seg); return DictDoubleArray.of(DType.F64, codes.length, values, randomCodesStructure(random, codes)); @@ -340,7 +340,7 @@ private static Array dictOfDoubles(Random random, int poolSize, int[] codes) { private static Array dictOfFloats(Random random, int poolSize, int[] codes) { MemorySegment seg = ARENA.allocate(Math.max(4L, poolSize * 4L), 4); for (int i = 0; i < poolSize; i++) { - seg.setAtIndex(PTypeIO.LE_FLOAT, i, random.nextInt(101) - 50); + seg.setAtIndex(VortexFormat.LE_FLOAT, i, random.nextInt(101) - 50); } MaterializedFloatArray values = new MaterializedFloatArray(DType.F32, poolSize, seg); return DictFloatArray.of(DType.F32, codes.length, values, randomCodesStructure(random, codes)); @@ -384,7 +384,7 @@ private static Array randomAggColumn(Random random, int n) { } else { MemorySegment seg = ARENA.allocate(Math.max(8L, n * 8L), 8); for (int i = 0; i < n; i++) { - seg.setAtIndex(PTypeIO.LE_DOUBLE, i, random.nextInt(1001) - 500); + seg.setAtIndex(VortexFormat.LE_DOUBLE, i, random.nextInt(1001) - 500); } agg = new MaterializedDoubleArray(DType.F64, n, seg); } @@ -428,7 +428,7 @@ private static DictLongArray flatDict(long[] pool, int[] codes) { private static DictDoubleArray flatDoubleDict(double[] pool, int[] codes) { MemorySegment seg = ARENA.allocate(Math.max(8L, pool.length * 8L), 8); for (int i = 0; i < pool.length; i++) { - seg.setAtIndex(PTypeIO.LE_DOUBLE, i, pool[i]); + seg.setAtIndex(VortexFormat.LE_DOUBLE, i, pool[i]); } MaterializedDoubleArray values = new MaterializedDoubleArray(DType.F64, pool.length, seg); return DictDoubleArray.of(DType.F64, codes.length, values, byteCodes(codes)); @@ -443,7 +443,7 @@ private static DictLongArray dictWithChunkedCodes(long[] pool, int[] first, int[ private static MaterializedLongArray longPool(long[] pool) { MemorySegment seg = ARENA.allocate(Math.max(8L, pool.length * 8L), 8); for (int i = 0; i < pool.length; i++) { - seg.setAtIndex(PTypeIO.LE_LONG, i, pool[i]); + seg.setAtIndex(VortexFormat.LE_LONG, i, pool[i]); } return new MaterializedLongArray(DType.I64, pool.length, seg); } @@ -451,7 +451,7 @@ private static MaterializedLongArray longPool(long[] pool) { private static MaterializedLongArray longAgg(long... values) { MemorySegment seg = ARENA.allocate(Math.max(8L, values.length * 8L), 8); for (int i = 0; i < values.length; i++) { - seg.setAtIndex(PTypeIO.LE_LONG, i, values[i]); + seg.setAtIndex(VortexFormat.LE_LONG, i, values[i]); } return new MaterializedLongArray(DType.I64, values.length, seg); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/FusedFilterSumTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/FusedFilterSumTest.java index 37fc517d..e8469a9a 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/FusedFilterSumTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/FusedFilterSumTest.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.reader.compute; import io.github.dfa1.vortex.core.error.VortexException; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.MaskedArray; @@ -286,7 +286,7 @@ private static boolean[] randomValidity(Random random, int n) { private static Array longColumn(long[] values, boolean[] valid) { MemorySegment seg = ARENA.allocate(Math.max(8L, values.length * 8L), 8); for (int i = 0; i < values.length; i++) { - seg.setAtIndex(PTypeIO.LE_LONG, i, values[i]); + seg.setAtIndex(VortexFormat.LE_LONG, i, values[i]); } return wrap(new MaterializedLongArray(DType.I64, values.length, seg), valid); } @@ -294,7 +294,7 @@ private static Array longColumn(long[] values, boolean[] valid) { private static Array intColumn(long[] values, boolean[] valid) { MemorySegment seg = ARENA.allocate(Math.max(4L, values.length * 4L), 4); for (int i = 0; i < values.length; i++) { - seg.setAtIndex(PTypeIO.LE_INT, i, (int) values[i]); + seg.setAtIndex(VortexFormat.LE_INT, i, (int) values[i]); } return wrap(new MaterializedIntArray(DType.I32, values.length, seg), valid); } @@ -302,7 +302,7 @@ private static Array intColumn(long[] values, boolean[] valid) { private static Array shortColumn(long[] values, boolean[] valid, boolean unsigned) { MemorySegment seg = ARENA.allocate(Math.max(2L, values.length * 2L), 2); for (int i = 0; i < values.length; i++) { - seg.setAtIndex(PTypeIO.LE_SHORT, i, (short) values[i]); + seg.setAtIndex(VortexFormat.LE_SHORT, i, (short) values[i]); } return wrap(new MaterializedShortArray(unsigned ? DType.U16 : DType.I16, values.length, seg), valid); } @@ -318,7 +318,7 @@ private static Array byteColumn(long[] values, boolean[] valid, boolean unsigned private static Array floatColumn(double[] values, boolean[] valid) { MemorySegment seg = ARENA.allocate(Math.max(4L, values.length * 4L), 4); for (int i = 0; i < values.length; i++) { - seg.setAtIndex(PTypeIO.LE_FLOAT, i, (float) values[i]); + seg.setAtIndex(VortexFormat.LE_FLOAT, i, (float) values[i]); } return wrap(new MaterializedFloatArray(DType.F32, values.length, seg), valid); } @@ -326,7 +326,7 @@ private static Array floatColumn(double[] values, boolean[] valid) { private static Array doubleColumn(double[] values, boolean[] valid) { MemorySegment seg = ARENA.allocate(Math.max(8L, values.length * 8L), 8); for (int i = 0; i < values.length; i++) { - seg.setAtIndex(PTypeIO.LE_DOUBLE, i, values[i]); + seg.setAtIndex(VortexFormat.LE_DOUBLE, i, values[i]); } return wrap(new MaterializedDoubleArray(DType.F64, values.length, seg), valid); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ValuesTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ValuesTest.java index 12e8742d..04f8f9a1 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ValuesTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/compute/ValuesTest.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.reader.compute; import io.github.dfa1.vortex.core.error.VortexException; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.GenericArray; @@ -67,7 +67,7 @@ void extensionDtypePassesIntegerThrough() { // widening must pass the raw value through rather than assume a primitive ptype DType extension = new DType.Extension("test.ext", DType.I32, null, false); MemorySegment seg = ARENA.allocate(4, 4); - seg.setAtIndex(PTypeIO.LE_INT, 0, -7); + seg.setAtIndex(VortexFormat.LE_INT, 0, -7); Array column = new MaterializedIntArray(extension, 1, seg); // When the value is boxed @@ -130,7 +130,7 @@ private static Array byteArray(DType dtype, byte... values) { private static Array shortArray(DType dtype, short... values) { MemorySegment seg = ARENA.allocate(Math.max(2L, values.length * 2L), 2); for (int i = 0; i < values.length; i++) { - seg.setAtIndex(PTypeIO.LE_SHORT, i, values[i]); + seg.setAtIndex(VortexFormat.LE_SHORT, i, values[i]); } return new MaterializedShortArray(dtype, values.length, seg); } @@ -138,7 +138,7 @@ private static Array shortArray(DType dtype, short... values) { private static Array intArray(DType dtype, int... values) { MemorySegment seg = ARENA.allocate(Math.max(4L, values.length * 4L), 4); for (int i = 0; i < values.length; i++) { - seg.setAtIndex(PTypeIO.LE_INT, i, values[i]); + seg.setAtIndex(VortexFormat.LE_INT, i, values[i]); } return new MaterializedIntArray(dtype, values.length, seg); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoderTest.java index 973d69fb..5a985ce3 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/DeltaEncodingDecoderTest.java @@ -3,7 +3,7 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.encoding.TestSegments; import io.github.dfa1.vortex.core.proto.ProtoDeltaMetadata; import io.github.dfa1.vortex.reader.ReadRegistry; @@ -97,6 +97,6 @@ void decode_constantBases_nonZeroOffsetAndBase() { assertThat(result.getLong(0)).isEqualTo(5L); // sanity: materialized bytes are little-endian MemorySegment seg = result.materialize(Arena.ofAuto()); - assertThat(seg.get(PTypeIO.LE_LONG, 0)).isEqualTo(5L); + assertThat(seg.get(VortexFormat.LE_LONG, 0)).isEqualTo(5L); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PcoTansDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PcoTansDecoderTest.java index 4eac89a8..05fd8274 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PcoTansDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/PcoTansDecoderTest.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.reader.decode; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -75,7 +75,7 @@ void singleBin_zeroBits_constantOutput() { // Then — all raw latent values = 42 for (int i = 0; i < n; i++) { - assertThat(out.get(PTypeIO.LE_LONG, (long) i * Long.BYTES)) + assertThat(out.get(VortexFormat.LE_LONG, (long) i * Long.BYTES)) .as("latent[%d]", i) .isEqualTo(42L); } @@ -101,7 +101,7 @@ void singleBin_oneBitOffset_decodesOffset() { // Then — offsets all zero → all values = lower + 0 = 10 for (int i = 0; i < n; i++) { - assertThat(out.get(PTypeIO.LE_LONG, (long) i * Long.BYTES)) + assertThat(out.get(VortexFormat.LE_LONG, (long) i * Long.BYTES)) .as("latent[%d]", i) .isEqualTo(10L); } @@ -125,7 +125,7 @@ void degenerateBins_zeroBins_outputZero() { // Then — all zero for (int i = 0; i < n; i++) { - assertThat(out.get(PTypeIO.LE_LONG, (long) i * Long.BYTES)).isZero(); + assertThat(out.get(VortexFormat.LE_LONG, (long) i * Long.BYTES)).isZero(); } } @@ -157,7 +157,7 @@ void nonDegenerate_ansSizeLog1_twoBins_stateTransitionsStayInBounds() { // Then — all states started at 0 → sym=0 → lower=5 for every value for (int i = 0; i < n; i++) { - assertThat(out.get(PTypeIO.LE_LONG, (long) i * Long.BYTES)) + assertThat(out.get(VortexFormat.LE_LONG, (long) i * Long.BYTES)) .as("latent[%d]", i) .isEqualTo(5L); } @@ -182,7 +182,7 @@ void moreThanOneBatch_decodesCorrectly() { // Then — all values = 7 for (int i = 0; i < n; i++) { - assertThat(out.get(PTypeIO.LE_LONG, (long) i * Long.BYTES)) + assertThat(out.get(VortexFormat.LE_LONG, (long) i * Long.BYTES)) .as("latent[%d]", i) .isEqualTo(7L); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/SegmentBroadcastTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/SegmentBroadcastTest.java index 5ee33dd3..7080baea 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/SegmentBroadcastTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/SegmentBroadcastTest.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.reader.decode; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import org.junit.jupiter.api.Test; import java.lang.foreign.Arena; @@ -18,7 +18,7 @@ void elementOffset_wrapsConstantSegment() { try (Arena arena = Arena.ofConfined()) { // Given MemorySegment seg = arena.allocate(8); - seg.set(PTypeIO.LE_LONG, 0, 99L); + seg.set(VortexFormat.LE_LONG, 0, 99L); // When / Then assertThat(SegmentBroadcast.elementOffset(seg, 0, 8)).isZero(); @@ -48,7 +48,7 @@ void broadcastCopy_replicatesSingleElement() { try (Arena arena = Arena.ofConfined()) { // Given MemorySegment src = arena.allocate(8); - src.set(PTypeIO.LE_LONG, 0, 0xCAFEBABEL); + src.set(VortexFormat.LE_LONG, 0, 0xCAFEBABEL); MemorySegment dst = arena.allocate(8 * 5); // When @@ -56,7 +56,7 @@ void broadcastCopy_replicatesSingleElement() { // Then for (long i = 0; i < 5; i++) { - assertThat(dst.getAtIndex(PTypeIO.LE_LONG, i)).isEqualTo(0xCAFEBABEL); + assertThat(dst.getAtIndex(VortexFormat.LE_LONG, i)).isEqualTo(0xCAFEBABEL); } } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VarBinViewEncodingDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VarBinViewEncodingDecoderTest.java index 8efc8dd9..4c7fe92a 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VarBinViewEncodingDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/decode/VarBinViewEncodingDecoderTest.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.array.VarBinArray; @@ -77,7 +77,7 @@ void decode_noBuffers_throws() { /// Writes a ≤12-byte inline view: length prefix then the bytes packed in-place. private static void writeInlineView(MemorySegment views, int row, byte[] bytes) { long off = (long) row * 16; - views.set(PTypeIO.LE_INT, off, bytes.length); + views.set(VortexFormat.LE_INT, off, bytes.length); MemorySegment.copy(MemorySegment.ofArray(bytes), 0, views, off + 4, bytes.length); } } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionTestSupport.java b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionTestSupport.java index d6776e76..431067ab 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionTestSupport.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/ExtensionTestSupport.java @@ -6,7 +6,7 @@ import io.github.dfa1.vortex.reader.array.MaterializedIntArray; import io.github.dfa1.vortex.reader.array.MaterializedLongArray; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_SHORT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_SHORT; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimestampExtensionDecoderTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimestampExtensionDecoderTest.java index 3b75e912..64796268 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimestampExtensionDecoderTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/extension/TimestampExtensionDecoderTest.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.reader.extension; + import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.TimeUnit; @@ -46,7 +47,7 @@ void dtype_defaultIsMsUtcless() { // Then — byte 0 = ms ordinal, bytes 1..3 = 0 (tz_len = 0) assertThat(dtype.storageDType()).isEqualTo(DType.I64); assertThat(dtype.metadata().get(java.lang.foreign.ValueLayout.JAVA_BYTE, 0)).isEqualTo((byte) TimeUnit.Milliseconds.ordinal()); - assertThat(dtype.metadata().get(io.github.dfa1.vortex.core.io.PTypeIO.LE_SHORT, 1)).isEqualTo((short) 0); + assertThat(dtype.metadata().get(io.github.dfa1.vortex.core.io.VortexFormat.LE_SHORT, 1)).isEqualTo((short) 0); } @Test @@ -55,7 +56,7 @@ void dtype_withTimezoneEncodesIanaName() { DType.Extension dtype = sut.dtype(TimeUnit.Microseconds, ZoneId.of("Europe/Paris"), false); // Then — header tz_len matches the UTF-8 length; the actual bytes follow - int tzLen = Short.toUnsignedInt(dtype.metadata().get(io.github.dfa1.vortex.core.io.PTypeIO.LE_SHORT, 1)); + int tzLen = Short.toUnsignedInt(dtype.metadata().get(io.github.dfa1.vortex.core.io.VortexFormat.LE_SHORT, 1)); assertThat(tzLen).isEqualTo("Europe/Paris".getBytes().length); assertThat(sut.timezone(dtype)).contains(ZoneId.of("Europe/Paris")); } diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchemaTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchemaTest.java index e0665b32..4584ea8b 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchemaTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/layout/ZonedStatsSchemaTest.java @@ -1,5 +1,6 @@ package io.github.dfa1.vortex.reader.layout; + import java.lang.foreign.MemorySegment; import io.github.dfa1.vortex.core.model.DType; @@ -20,7 +21,7 @@ class ZoneLength { void readsLittleEndianU32() { // Given — 8192 = 0x2000 stored as LE u32 MemorySegment meta = MemorySegment.ofArray(new byte[4]); - meta.set(io.github.dfa1.vortex.core.io.PTypeIO.LE_INT, 0, 8192); + meta.set(io.github.dfa1.vortex.core.io.VortexFormat.LE_INT, 0, 8192); // When long result = ZonedStatsSchema.zoneLength(meta); @@ -71,7 +72,7 @@ void decodesMultiByteBitset() { // byte 0: 0b0101_1000 = 0x58 (bits 3,4,6) // byte 1: 0b0000_0001 = 0x01 (bit 8 → bit 0 of second byte) MemorySegment meta = MemorySegment.ofArray(new byte[4 + 2]); - meta.set(io.github.dfa1.vortex.core.io.PTypeIO.LE_INT, 0, 8192); + meta.set(io.github.dfa1.vortex.core.io.VortexFormat.LE_INT, 0, 8192); meta.set(java.lang.foreign.ValueLayout.JAVA_BYTE, 4, (byte) 0x58); meta.set(java.lang.foreign.ValueLayout.JAVA_BYTE, 5, (byte) 0x01); @@ -103,7 +104,7 @@ void ignoresFutureStatBits() { // Given — bit 31 set (beyond any known stat) plus MAX/MIN // byte 0: 0x18 (MAX|MIN), bytes 1-2: 0, byte 3: 0x80 (bit 31) MemorySegment meta = MemorySegment.ofArray(new byte[4 + 4]); - meta.set(io.github.dfa1.vortex.core.io.PTypeIO.LE_INT, 0, 8192); + meta.set(io.github.dfa1.vortex.core.io.VortexFormat.LE_INT, 0, 8192); meta.set(java.lang.foreign.ValueLayout.JAVA_BYTE, 4, (byte) 0x18); meta.set(java.lang.foreign.ValueLayout.JAVA_BYTE, 7, (byte) 0x80); @@ -246,7 +247,7 @@ void allStatsTogetherForI32() { private static MemorySegment metaWithBitset(int zoneLen, int firstByte) { MemorySegment meta = MemorySegment.ofArray(new byte[4 + 1]); - meta.set(io.github.dfa1.vortex.core.io.PTypeIO.LE_INT, 0, zoneLen); + meta.set(io.github.dfa1.vortex.core.io.VortexFormat.LE_INT, 0, zoneLen); meta.set(java.lang.foreign.ValueLayout.JAVA_BYTE, 4, (byte) firstByte); return meta; } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpEncodingEncoder.java index d929646e..df51216c 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/AlpEncodingEncoder.java @@ -3,7 +3,7 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoALPMetadata; import io.github.dfa1.vortex.core.proto.ProtoPatchesMetadata; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; @@ -196,7 +196,7 @@ private static EncodeResult encodeF64(double[] values, EncodeContext ctx) { MemorySegment encodedBuf = ctx.arena().allocate((long) n * 8, 8); for (int i = 0; i < n; i++) { - encodedBuf.setAtIndex(PTypeIO.LE_LONG, i, d.encodedArr()[i]); + encodedBuf.setAtIndex(VortexFormat.LE_LONG, i, d.encodedArr()[i]); } EncodeNode encodedNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 0); @@ -212,8 +212,8 @@ private static EncodeResult encodeF64(double[] values, EncodeContext ctx) { MemorySegment idxBuf = ctx.arena().allocate((long) numPatches * 4, 4); MemorySegment valBuf = ctx.arena().allocate((long) numPatches * 8, 8); for (int i = 0; i < numPatches; i++) { - idxBuf.setAtIndex(PTypeIO.LE_INT, i, d.patchIndices().get(i)); - valBuf.setAtIndex(PTypeIO.LE_DOUBLE, i, d.patchValues().get(i)); + idxBuf.setAtIndex(VortexFormat.LE_INT, i, d.patchIndices().get(i)); + valBuf.setAtIndex(VortexFormat.LE_DOUBLE, i, d.patchValues().get(i)); } ProtoPatchesMetadata patches = buildPatchesMeta(numPatches); @@ -242,8 +242,8 @@ private static CascadeStep encodeCascadeF64(double[] values, EncodeContext ctx) MemorySegment idxBuf = ctx.arena().allocate((long) numPatches * 4, 4); MemorySegment valBuf = ctx.arena().allocate((long) numPatches * 8, 8); for (int i = 0; i < numPatches; i++) { - idxBuf.setAtIndex(PTypeIO.LE_INT, i, d.patchIndices().get(i)); - valBuf.setAtIndex(PTypeIO.LE_DOUBLE, i, d.patchValues().get(i)); + idxBuf.setAtIndex(VortexFormat.LE_INT, i, d.patchIndices().get(i)); + valBuf.setAtIndex(VortexFormat.LE_DOUBLE, i, d.patchValues().get(i)); } ProtoPatchesMetadata patches = buildPatchesMeta(numPatches); @@ -365,7 +365,7 @@ private static EncodeResult encodeF32(float[] values, EncodeContext ctx) { MemorySegment encodedBuf = ctx.arena().allocate((long) n * 4, 4); for (int i = 0; i < n; i++) { - encodedBuf.setAtIndex(PTypeIO.LE_INT, i, encodedArr[i]); + encodedBuf.setAtIndex(VortexFormat.LE_INT, i, encodedArr[i]); } EncodeNode encodedNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 0); @@ -381,8 +381,8 @@ private static EncodeResult encodeF32(float[] values, EncodeContext ctx) { MemorySegment idxBuf = ctx.arena().allocate((long) numPatches * 4, 4); MemorySegment valBuf = ctx.arena().allocate((long) numPatches * 4, 4); for (int i = 0; i < numPatches; i++) { - idxBuf.setAtIndex(PTypeIO.LE_INT, i, patchIndices.get(i)); - valBuf.setAtIndex(PTypeIO.LE_FLOAT, i, patchValues.get(i)); + idxBuf.setAtIndex(VortexFormat.LE_INT, i, patchIndices.get(i)); + valBuf.setAtIndex(VortexFormat.LE_FLOAT, i, patchValues.get(i)); } ProtoPatchesMetadata patches = new ProtoPatchesMetadata( diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoder.java index e184b7f7..8dd97162 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoder.java @@ -6,6 +6,7 @@ import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.compute.FastLanes; import io.github.dfa1.vortex.core.compute.PrimitiveArrays; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.io.PTypeIO; import io.github.dfa1.vortex.core.proto.ProtoBitPackedMetadata; import io.github.dfa1.vortex.core.proto.ProtoPatchesMetadata; @@ -250,9 +251,9 @@ private static byte[] statsBytes(PType ptype, long value) { private static long readWordFromSeg(MemorySegment seg, int off, int typeBits) { return switch (typeBits) { case 8 -> Byte.toUnsignedLong(seg.get(ValueLayout.JAVA_BYTE, off)); - case 16 -> Short.toUnsignedLong(seg.get(PTypeIO.LE_SHORT, off)); - case 32 -> Integer.toUnsignedLong(seg.get(PTypeIO.LE_INT, off)); - case 64 -> seg.get(PTypeIO.LE_LONG, off); + case 16 -> Short.toUnsignedLong(seg.get(VortexFormat.LE_SHORT, off)); + case 32 -> Integer.toUnsignedLong(seg.get(VortexFormat.LE_INT, off)); + case 64 -> seg.get(VortexFormat.LE_LONG, off); default -> throw new VortexException(EncodingId.FASTLANES_BITPACKED, "unsupported typeBits: " + typeBits); }; @@ -261,9 +262,9 @@ private static long readWordFromSeg(MemorySegment seg, int off, int typeBits) { private static void writeWordToSeg(MemorySegment seg, int off, long value, int typeBits) { switch (typeBits) { case 8 -> seg.set(ValueLayout.JAVA_BYTE, off, (byte) value); - case 16 -> seg.set(PTypeIO.LE_SHORT, off, (short) value); - case 32 -> seg.set(PTypeIO.LE_INT, off, (int) value); - case 64 -> seg.set(PTypeIO.LE_LONG, off, value); + case 16 -> seg.set(VortexFormat.LE_SHORT, off, (short) value); + case 32 -> seg.set(VortexFormat.LE_INT, off, (int) value); + case 64 -> seg.set(VortexFormat.LE_LONG, off, value); default -> throw new VortexException(EncodingId.FASTLANES_BITPACKED, "unsupported typeBits: " + typeBits); } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoder.java index 205645f5..2d44e946 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoder.java @@ -4,6 +4,7 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.io.PTypeIO; import io.github.dfa1.vortex.core.proto.ProtoDictMetadata; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; @@ -130,11 +131,11 @@ private static EncodeResult encodeUtf8(String[] strings, EncodeContext ctx) { MemorySegment dictOffsetsBuf = arena.allocate((long) (dictSize + 1) * Long.BYTES, Long.BYTES); long pos = 0; - dictOffsetsBuf.setAtIndex(PTypeIO.LE_LONG, 0, 0L); + dictOffsetsBuf.setAtIndex(VortexFormat.LE_LONG, 0, 0L); for (int i = 0; i < dictSize; i++) { MemorySegment.copy(MemorySegment.ofArray(dictByteArrays[i]), 0, dictBytesBuf, pos, dictByteArrays[i].length); pos += dictByteArrays[i].length; - dictOffsetsBuf.setAtIndex(PTypeIO.LE_LONG, (long) i + 1, pos); + dictOffsetsBuf.setAtIndex(VortexFormat.LE_LONG, (long) i + 1, pos); } MemorySegment codesBuf = arena.allocate((long) n * codeBytes); @@ -205,14 +206,14 @@ private static DictData buildDictData(DType dtype, Object data, EncodeContext ct case U16 -> { short[] a = new short[len]; for (int i = 0; i < len; i++) { - a[i] = codesBuf.get(PTypeIO.LE_SHORT, (long) i * 2); + a[i] = codesBuf.get(VortexFormat.LE_SHORT, (long) i * 2); } yield a; } default -> { int[] a = new int[len]; for (int i = 0; i < len; i++) { - a[i] = codesBuf.get(PTypeIO.LE_INT, (long) i * 4); + a[i] = codesBuf.get(VortexFormat.LE_INT, (long) i * 4); } yield a; } @@ -317,8 +318,8 @@ private static Object buildUniqueArray(PType ptype, Iterable uniques, in private static void writeCodeToSeg(MemorySegment seg, PType codePType, int idx, int code) { switch (codePType) { case U8 -> seg.set(ValueLayout.JAVA_BYTE, idx, (byte) code); - case U16 -> seg.set(PTypeIO.LE_SHORT, (long) idx * 2, (short) code); - case U32 -> seg.set(PTypeIO.LE_INT, (long) idx * 4, code); + case U16 -> seg.set(VortexFormat.LE_SHORT, (long) idx * 2, (short) code); + case U32 -> seg.set(VortexFormat.LE_INT, (long) idx * 4, code); default -> throw new VortexException(EncodingId.VORTEX_DICT, "unexpected code type: " + codePType); } } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java index 3468fe7f..f8de6268 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoder.java @@ -3,7 +3,7 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoFSSTMetadata; import java.lang.foreign.Arena; @@ -80,7 +80,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { MemorySegment symBuf = arena.allocate(Math.max(numSymbols * 8L, 1), 8); for (int i = 0; i < numSymbols; i++) { - symBuf.setAtIndex(PTypeIO.LE_LONG, i, symbolValues[i]); + symBuf.setAtIndex(VortexFormat.LE_LONG, i, symbolValues[i]); } MemorySegment symLenBuf = arena.allocate(Math.max(numSymbols, 1)); @@ -101,15 +101,15 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { MemorySegment uncompLenBuf = arena.allocate(Math.max(n * 4L, 1), 4); for (int i = 0; i < n; i++) { - uncompLenBuf.setAtIndex(PTypeIO.LE_INT, i, byteArrays[i].length); + uncompLenBuf.setAtIndex(VortexFormat.LE_INT, i, byteArrays[i].length); } MemorySegment codesOffBuf = arena.allocate((long) (n + 1) * 4, 4); long off = 0; - codesOffBuf.setAtIndex(PTypeIO.LE_INT, 0, 0); + codesOffBuf.setAtIndex(VortexFormat.LE_INT, 0, 0); for (int i = 0; i < n; i++) { off += compressed[i].length; - codesOffBuf.setAtIndex(PTypeIO.LE_INT, (long) i + 1, (int) off); + codesOffBuf.setAtIndex(VortexFormat.LE_INT, (long) i + 1, (int) off); } byte[] metaBytes = new ProtoFSSTMetadata( diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ListEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ListEncodingEncoder.java index 33a8992f..59e39998 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ListEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ListEncodingEncoder.java @@ -7,7 +7,7 @@ import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoListMetadata; @@ -52,7 +52,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { long nOffsets = ld.outerLen() + 1; MemorySegment offsetsBuf = ctx.arena().allocate(nOffsets * Long.BYTES, Long.BYTES); for (int i = 0; i < nOffsets; i++) { - offsetsBuf.setAtIndex(PTypeIO.LE_LONG, i, ld.offsets()[i]); + offsetsBuf.setAtIndex(VortexFormat.LE_LONG, i, ld.offsets()[i]); } allBuffers.add(offsetsBuf); EncodeNode offsetsNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, elemBufCount); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ListViewEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ListViewEncodingEncoder.java index b36f7937..ea184c8b 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ListViewEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ListViewEncodingEncoder.java @@ -7,7 +7,7 @@ import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoListViewMetadata; @@ -53,14 +53,14 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { MemorySegment offsetsBuf = ctx.arena().allocate(n * Integer.BYTES, Integer.BYTES); for (int i = 0; i < n; i++) { - offsetsBuf.setAtIndex(PTypeIO.LE_INT, i, lvd.offsets()[i]); + offsetsBuf.setAtIndex(VortexFormat.LE_INT, i, lvd.offsets()[i]); } allBuffers.add(offsetsBuf); EncodeNode offsetsNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, elemBufCount); MemorySegment sizesBuf = ctx.arena().allocate(n * Integer.BYTES, Integer.BYTES); for (int i = 0; i < n; i++) { - sizesBuf.setAtIndex(PTypeIO.LE_INT, i, lvd.sizes()[i]); + sizesBuf.setAtIndex(VortexFormat.LE_INT, i, lvd.sizes()[i]); } allBuffers.add(sizesBuf); EncodeNode sizesNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, elemBufCount + 1); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PatchedEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PatchedEncodingEncoder.java index 021b20fa..29214702 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PatchedEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PatchedEncodingEncoder.java @@ -6,6 +6,7 @@ import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.compute.FastLanes; import io.github.dfa1.vortex.core.compute.PrimitiveArrays; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.io.PTypeIO; import io.github.dfa1.vortex.core.proto.ProtoPatchedMetadata; @@ -114,12 +115,12 @@ static EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { MemorySegment laneOffsBuf = ctx.arena().allocate(pd.laneOffsets.length * 4L); for (int i = 0; i < pd.laneOffsets.length; i++) { - laneOffsBuf.setAtIndex(PTypeIO.LE_INT, i, pd.laneOffsets[i]); + laneOffsBuf.setAtIndex(VortexFormat.LE_INT, i, pd.laneOffsets[i]); } MemorySegment patchIdxBuf = ctx.arena().allocate(Math.max(1L, (long) pd.nPatches * 2)); for (int i = 0; i < pd.nPatches; i++) { - patchIdxBuf.setAtIndex(PTypeIO.LE_SHORT, i, pd.patchIndices[i]); + patchIdxBuf.setAtIndex(VortexFormat.LE_SHORT, i, pd.patchIndices[i]); } MemorySegment patchValBuf = ctx.arena().allocate(Math.max(1L, (long) pd.nPatches * elemBytes)); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java index 094f19aa..3d6d62e5 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoder.java @@ -3,7 +3,7 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import java.lang.foreign.Arena; @@ -47,7 +47,7 @@ private static MemorySegment encodePrimitive(PType ptype, Object data, Arena are short[] arr = (short[]) data; MemorySegment seg = arena.allocate((long) arr.length * 2, 2); for (int i = 0; i < arr.length; i++) { - seg.setAtIndex(PTypeIO.LE_SHORT, i, arr[i]); + seg.setAtIndex(VortexFormat.LE_SHORT, i, arr[i]); } yield seg; } @@ -55,7 +55,7 @@ private static MemorySegment encodePrimitive(PType ptype, Object data, Arena are int[] arr = (int[]) data; MemorySegment seg = arena.allocate((long) arr.length * 4, 4); for (int i = 0; i < arr.length; i++) { - seg.setAtIndex(PTypeIO.LE_INT, i, arr[i]); + seg.setAtIndex(VortexFormat.LE_INT, i, arr[i]); } yield seg; } @@ -63,7 +63,7 @@ private static MemorySegment encodePrimitive(PType ptype, Object data, Arena are long[] arr = (long[]) data; MemorySegment seg = arena.allocate((long) arr.length * 8, 8); for (int i = 0; i < arr.length; i++) { - seg.setAtIndex(PTypeIO.LE_LONG, i, arr[i]); + seg.setAtIndex(VortexFormat.LE_LONG, i, arr[i]); } yield seg; } @@ -71,7 +71,7 @@ private static MemorySegment encodePrimitive(PType ptype, Object data, Arena are float[] arr = (float[]) data; MemorySegment seg = arena.allocate((long) arr.length * 4, 4); for (int i = 0; i < arr.length; i++) { - seg.setAtIndex(PTypeIO.LE_FLOAT, i, arr[i]); + seg.setAtIndex(VortexFormat.LE_FLOAT, i, arr[i]); } yield seg; } @@ -79,7 +79,7 @@ private static MemorySegment encodePrimitive(PType ptype, Object data, Arena are double[] arr = (double[]) data; MemorySegment seg = arena.allocate((long) arr.length * 8, 8); for (int i = 0; i < arr.length; i++) { - seg.setAtIndex(PTypeIO.LE_DOUBLE, i, arr[i]); + seg.setAtIndex(VortexFormat.LE_DOUBLE, i, arr[i]); } yield seg; } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoder.java index 40a9025f..7d61ed63 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RleEncodingEncoder.java @@ -5,6 +5,7 @@ import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; import io.github.dfa1.vortex.core.compute.PrimitiveArrays; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.io.PTypeIO; import io.github.dfa1.vortex.core.proto.ProtoRLEMetadata; @@ -190,7 +191,7 @@ private static MemorySegment fromLongs(long[] values, int count, PType ptype, Se private static MemorySegment fromLongsU64(long[] values, int count, SegmentAllocator arena) { MemorySegment seg = arena.allocate((long) count * 8); for (int i = 0; i < count; i++) { - seg.setAtIndex(PTypeIO.LE_LONG, i, values[i]); + seg.setAtIndex(VortexFormat.LE_LONG, i, values[i]); } return seg; } @@ -198,7 +199,7 @@ private static MemorySegment fromLongsU64(long[] values, int count, SegmentAlloc private static MemorySegment toIndicesSeg(short[] indices, int count, SegmentAllocator arena) { MemorySegment seg = arena.allocate((long) count * 2); for (int i = 0; i < count; i++) { - seg.setAtIndex(PTypeIO.LE_SHORT, i, indices[i]); + seg.setAtIndex(VortexFormat.LE_SHORT, i, indices[i]); } return seg; } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java index e56b612e..43922707 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/RunEndEncodingEncoder.java @@ -4,6 +4,7 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.io.PTypeIO; import io.github.dfa1.vortex.core.proto.ProtoRunEndMetadata; @@ -80,7 +81,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { MemorySegment endsBuf = ctx.arena().allocate((long) numRuns * 4, 4); for (int i = 0; i < numRuns; i++) { - endsBuf.setAtIndex(PTypeIO.LE_INT, i, ends.get(i)); + endsBuf.setAtIndex(VortexFormat.LE_INT, i, ends.get(i)); } int elemBytes = ptype.byteSize(); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoder.java index 527ee021..d03e5721 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinEncodingEncoder.java @@ -3,7 +3,7 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoScalarValue; import io.github.dfa1.vortex.core.proto.ProtoVarBinMetadata; @@ -50,11 +50,11 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { MemorySegment offsetsBuf = arena.allocate((long) (n + 1) * Long.BYTES, Long.BYTES); long pos = 0; - offsetsBuf.setAtIndex(PTypeIO.LE_LONG, 0, 0L); + offsetsBuf.setAtIndex(VortexFormat.LE_LONG, 0, 0L); for (int i = 0; i < n; i++) { MemorySegment.copy(MemorySegment.ofArray(byteArrays[i]), 0, bytesBuf, pos, byteArrays[i].length); pos += byteArrays[i].length; - offsetsBuf.setAtIndex(PTypeIO.LE_LONG, (long) i + 1, pos); + offsetsBuf.setAtIndex(VortexFormat.LE_LONG, (long) i + 1, pos); } byte[] metaBytes = new ProtoVarBinMetadata(io.github.dfa1.vortex.core.proto.ProtoPType.fromValue(PType.I64.ordinal())).encode(); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoder.java index dd09b6d2..7f0c356c 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoder.java @@ -2,7 +2,7 @@ import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; @@ -52,13 +52,13 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { for (int i = 0; i < n; i++) { byte[] b = bytes[i]; long viewOff = (long) i * VIEW_SIZE; - viewsBuf.set(PTypeIO.LE_INT, viewOff, b.length); + viewsBuf.set(VortexFormat.LE_INT, viewOff, b.length); if (b.length <= MAX_INLINED_SIZE) { MemorySegment.copy(MemorySegment.ofArray(b), 0, viewsBuf, viewOff + 4, b.length); } else { MemorySegment.copy(MemorySegment.ofArray(b), 0, viewsBuf, viewOff + 4, 4); - viewsBuf.set(PTypeIO.LE_INT, viewOff + 8, 0); - viewsBuf.set(PTypeIO.LE_INT, viewOff + 12, dataOffset); + viewsBuf.set(VortexFormat.LE_INT, viewOff + 8, 0); + viewsBuf.set(VortexFormat.LE_INT, viewOff + 12, dataOffset); MemorySegment.copy(MemorySegment.ofArray(b), 0, dataBuf, dataOffset, b.length); dataOffset += b.length; } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoder.java index c9be756a..d3acf8ab 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZigZagEncodingEncoder.java @@ -4,7 +4,7 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; @@ -49,7 +49,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { MemorySegment s = ctx.arena().allocate((long) arr.length * 2, 2); for (int i = 0; i < arr.length; i++) { short v = arr[i]; - s.setAtIndex(PTypeIO.LE_SHORT, i, (short) ((v << 1) ^ (v >> 15))); + s.setAtIndex(VortexFormat.LE_SHORT, i, (short) ((v << 1) ^ (v >> 15))); } yield s; } @@ -58,7 +58,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { MemorySegment s = ctx.arena().allocate((long) arr.length * 4, 4); for (int i = 0; i < arr.length; i++) { int v = arr[i]; - s.setAtIndex(PTypeIO.LE_INT, i, (v << 1) ^ (v >> 31)); + s.setAtIndex(VortexFormat.LE_INT, i, (v << 1) ^ (v >> 31)); } yield s; } @@ -67,7 +67,7 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) { MemorySegment s = ctx.arena().allocate((long) arr.length * 8, 8); for (int i = 0; i < arr.length; i++) { long v = arr[i]; - s.setAtIndex(PTypeIO.LE_LONG, i, (v << 1) ^ (v >> 63)); + s.setAtIndex(VortexFormat.LE_LONG, i, (v << 1) ^ (v >> 63)); } yield s; } diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java index 6d36f9b3..f50247c6 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/encode/ZstdEncodingEncoder.java @@ -5,7 +5,7 @@ import io.github.dfa1.vortex.core.model.PType; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.proto.ProtoZstdFrameMetadata; import io.github.dfa1.vortex.core.proto.ProtoZstdMetadata; @@ -226,7 +226,7 @@ private FrameLayout varBinLayout(MemorySegment raw, long nValues) { long count = Math.min(valuesPerFrame, nValues - valueIdx); long start = pos; for (long k = 0; k < count; k++) { - int len = raw.get(PTypeIO.LE_INT, pos); + int len = raw.get(VortexFormat.LE_INT, pos); pos += 4L + len; } byteLengths[f] = pos - start; @@ -319,7 +319,7 @@ private static MemorySegment primitiveToLeBytes(PType ptype, Object data, Arena short[] arr = (short[]) data; MemorySegment seg = arena.allocate((long) arr.length * 2, 2); for (int i = 0; i < arr.length; i++) { - seg.setAtIndex(PTypeIO.LE_SHORT, i, arr[i]); + seg.setAtIndex(VortexFormat.LE_SHORT, i, arr[i]); } yield seg; } @@ -327,7 +327,7 @@ private static MemorySegment primitiveToLeBytes(PType ptype, Object data, Arena int[] arr = (int[]) data; MemorySegment seg = arena.allocate((long) arr.length * 4, 4); for (int i = 0; i < arr.length; i++) { - seg.setAtIndex(PTypeIO.LE_INT, i, arr[i]); + seg.setAtIndex(VortexFormat.LE_INT, i, arr[i]); } yield seg; } @@ -335,7 +335,7 @@ private static MemorySegment primitiveToLeBytes(PType ptype, Object data, Arena long[] arr = (long[]) data; MemorySegment seg = arena.allocate((long) arr.length * 8, 8); for (int i = 0; i < arr.length; i++) { - seg.setAtIndex(PTypeIO.LE_LONG, i, arr[i]); + seg.setAtIndex(VortexFormat.LE_LONG, i, arr[i]); } yield seg; } @@ -343,7 +343,7 @@ private static MemorySegment primitiveToLeBytes(PType ptype, Object data, Arena float[] arr = (float[]) data; MemorySegment seg = arena.allocate((long) arr.length * 4, 4); for (int i = 0; i < arr.length; i++) { - seg.setAtIndex(PTypeIO.LE_FLOAT, i, arr[i]); + seg.setAtIndex(VortexFormat.LE_FLOAT, i, arr[i]); } yield seg; } @@ -351,7 +351,7 @@ private static MemorySegment primitiveToLeBytes(PType ptype, Object data, Arena double[] arr = (double[]) data; MemorySegment seg = arena.allocate((long) arr.length * 8, 8); for (int i = 0; i < arr.length; i++) { - seg.setAtIndex(PTypeIO.LE_DOUBLE, i, arr[i]); + seg.setAtIndex(VortexFormat.LE_DOUBLE, i, arr[i]); } yield seg; } @@ -379,7 +379,7 @@ private static MemorySegment buildLengthPrefixed(String[] strings, Arena arena) MemorySegment seg = arena.allocate(total > 0 ? total : 1); long pos = 0; for (byte[] bytes : encoded) { - seg.set(PTypeIO.LE_INT, pos, bytes.length); + seg.set(VortexFormat.LE_INT, pos, bytes.length); pos += 4; MemorySegment.copy(MemorySegment.ofArray(bytes), 0, seg, pos, bytes.length); pos += bytes.length; diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/DictEncodingTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/DictEncodingTest.java index b161d1a4..99f3182c 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/DictEncodingTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/DictEncodingTest.java @@ -1,6 +1,6 @@ package io.github.dfa1.vortex.writer; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.DType; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.writer.encode.DictEncodingEncoder; @@ -96,21 +96,21 @@ void roundTrip_multipleChunks(@TempDir Path tmp) throws IOException { var iter = vf.scan(ScanOptions.all())) { assertThat(iter.hasNext()).isTrue(); try (Chunk c1 = iter.next()) { - Array a1 = c1.columns().get("category"); + Array a1 = c1.column("category"); assertThat(a1.length()).isEqualTo(3L); - assertThat(a1.materialize(Arena.ofAuto()).get(PTypeIO.LE_INT, 0)).isEqualTo(10); - assertThat(a1.materialize(Arena.ofAuto()).get(PTypeIO.LE_INT, 4)).isEqualTo(20); - assertThat(a1.materialize(Arena.ofAuto()).get(PTypeIO.LE_INT, 8)).isEqualTo(10); + assertThat(a1.materialize(Arena.ofAuto()).get(VortexFormat.LE_INT, 0)).isEqualTo(10); + assertThat(a1.materialize(Arena.ofAuto()).get(VortexFormat.LE_INT, 4)).isEqualTo(20); + assertThat(a1.materialize(Arena.ofAuto()).get(VortexFormat.LE_INT, 8)).isEqualTo(10); } assertThat(iter.hasNext()).isTrue(); try (Chunk c2 = iter.next()) { - Array a2 = c2.columns().get("category"); + Array a2 = c2.column("category"); assertThat(a2.length()).isEqualTo(4L); - assertThat(a2.materialize(Arena.ofAuto()).get(PTypeIO.LE_INT, 0)).isEqualTo(30); - assertThat(a2.materialize(Arena.ofAuto()).get(PTypeIO.LE_INT, 4)).isEqualTo(10); - assertThat(a2.materialize(Arena.ofAuto()).get(PTypeIO.LE_INT, 8)).isEqualTo(20); - assertThat(a2.materialize(Arena.ofAuto()).get(PTypeIO.LE_INT, 12)).isEqualTo(30); + assertThat(a2.materialize(Arena.ofAuto()).get(VortexFormat.LE_INT, 0)).isEqualTo(30); + assertThat(a2.materialize(Arena.ofAuto()).get(VortexFormat.LE_INT, 4)).isEqualTo(10); + assertThat(a2.materialize(Arena.ofAuto()).get(VortexFormat.LE_INT, 8)).isEqualTo(20); + assertThat(a2.materialize(Arena.ofAuto()).get(VortexFormat.LE_INT, 12)).isEqualTo(30); } assertThat(iter.hasNext()).isFalse(); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictPrimitiveTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictPrimitiveTest.java index 199bc91f..5f03c688 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictPrimitiveTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/GlobalDictPrimitiveTest.java @@ -188,7 +188,7 @@ void lowCardinality_i16_notDicted_roundTrips(@TempDir Path tmp) throws IOExcepti var iter = vf.scan(io.github.dfa1.vortex.reader.ScanOptions.columns("v"))) { var out = new java.util.ArrayList(); iter.forEachRemaining(c -> { - var arr = (io.github.dfa1.vortex.reader.array.ShortArray) c.columns().get("v"); + io.github.dfa1.vortex.reader.array.ShortArray arr = c.column("v"); for (long i = 0; i < arr.length(); i++) { out.add(arr.getShort(i)); } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/VortexWriterTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/VortexWriterTest.java index e487cd4f..a87b3177 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/VortexWriterTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/VortexWriterTest.java @@ -39,7 +39,9 @@ private static List snapshotAll(VortexReader vf, ScanOptions opts var snapshots = new ArrayList(); try (var iter = vf.scan(opts)) { iter.forEachRemaining(c -> - snapshots.add(new ChunkSnapshot(c.rowCount(), java.util.Set.copyOf(c.columns().keySet())))); + snapshots.add(new ChunkSnapshot(c.rowCount(), c.columns().keySet().stream() + .map(io.github.dfa1.vortex.core.model.ColumnName::value) + .collect(java.util.stream.Collectors.toUnmodifiableSet())))); } return snapshots; } @@ -478,7 +480,7 @@ void writeAndRead_idValues_decodedCorrectly(@TempDir Path tmp) throws IOExceptio var iter = vf.scan(ScanOptions.all())) { assertThat(iter.hasNext()).isTrue(); try (Chunk chunk = iter.next()) { - LongArray idArray = (LongArray) chunk.columns().get("id"); + LongArray idArray = chunk.column("id"); assertThat(idArray.length()).isEqualTo(3L); assertThat(idArray.getLong(0)).isEqualTo(42L); assertThat(idArray.getLong(1)).isEqualTo(100L); diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/WriterZoneMapTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/WriterZoneMapTest.java index 61c82cdc..ee44d23d 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/WriterZoneMapTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/WriterZoneMapTest.java @@ -1,8 +1,8 @@ package io.github.dfa1.vortex.writer; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_FLOAT; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_INT; -import static io.github.dfa1.vortex.core.io.PTypeIO.LE_SHORT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_FLOAT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_INT; +import static io.github.dfa1.vortex.core.io.VortexFormat.LE_SHORT; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedConstantPatchesBroadcastTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedConstantPatchesBroadcastTest.java index c3a8cf0d..96445b62 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedConstantPatchesBroadcastTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedConstantPatchesBroadcastTest.java @@ -5,7 +5,7 @@ import io.github.dfa1.vortex.reader.decode.ArrayNode; import io.github.dfa1.vortex.reader.decode.DecodeContext; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.core.proto.ProtoBitPackedMetadata; import io.github.dfa1.vortex.core.proto.ProtoPatchesMetadata; @@ -70,12 +70,12 @@ void bitpackedDecode_withConstantPatchesValues_broadcastsValueAcrossPatches() { // Then assertThat(result.length()).isEqualTo(n); MemorySegment data = result.materialize(Arena.ofAuto()); - assertThat(data.getAtIndex(PTypeIO.LE_LONG, 2)).isEqualTo(constantPatchValue); + assertThat(data.getAtIndex(VortexFormat.LE_LONG, 2)).isEqualTo(constantPatchValue); for (long i = 0; i < n; i++) { if (i == 2) { continue; } - assertThat(data.getAtIndex(PTypeIO.LE_LONG, i)).as("non-patched index %d", i).isZero(); + assertThat(data.getAtIndex(VortexFormat.LE_LONG, i)).as("non-patched index %d", i).isZero(); } } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoderTest.java index dec4d11f..c4ba0be9 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingEncoderTest.java @@ -4,7 +4,7 @@ import io.github.dfa1.vortex.encoding.DTypes; import io.github.dfa1.vortex.reader.decode.DecodeContext; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; import io.github.dfa1.vortex.core.proto.ProtoBitPackedMetadata; @@ -56,7 +56,7 @@ void encodeDecode_u32_isLossless(String name, int[] data) { assertThat(result.length()).isEqualTo(data.length); for (int i = 0; i < data.length; i++) { - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_INT, (long) i * 4)).as("index %d", i).isEqualTo(data[i]); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_INT, (long) i * 4)).as("index %d", i).isEqualTo(data[i]); } } @@ -69,7 +69,7 @@ void encodeDecode_u64_isLossless(String name, long[] data) { assertThat(result.length()).isEqualTo(data.length); for (int i = 0; i < data.length; i++) { - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_LONG, (long) i * 8)).as("index %d", i).isEqualTo(data[i]); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_LONG, (long) i * 8)).as("index %d", i).isEqualTo(data[i]); } } @@ -98,7 +98,7 @@ void encodeDecode_u32_everyBitWidth(int width, int[] data) { var seg = result.materialize(Arena.ofAuto()); for (int i = 0; i < data.length; i++) { - assertThat(seg.get(PTypeIO.LE_INT, (long) i * 4)).as("width %d index %d", width, i).isEqualTo(data[i]); + assertThat(seg.get(VortexFormat.LE_INT, (long) i * 4)).as("width %d index %d", width, i).isEqualTo(data[i]); } } @@ -111,7 +111,7 @@ void encodeDecode_u64_everyBitWidth(int width, long[] data) { var seg = result.materialize(Arena.ofAuto()); for (int i = 0; i < data.length; i++) { - assertThat(seg.get(PTypeIO.LE_LONG, (long) i * 8)).as("width %d index %d", width, i).isEqualTo(data[i]); + assertThat(seg.get(VortexFormat.LE_LONG, (long) i * 8)).as("width %d index %d", width, i).isEqualTo(data[i]); } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingPatchesTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingPatchesTest.java index ed1438cd..963a91c4 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingPatchesTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/BitpackedEncodingPatchesTest.java @@ -6,7 +6,7 @@ import io.github.dfa1.vortex.reader.decode.DecodeContext; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; import io.github.dfa1.vortex.core.proto.ProtoBitPackedMetadata; @@ -73,11 +73,11 @@ void decode_appliesPatches_overridingBitPackedValues() { // Then assertThat(result.length()).isEqualTo(base.length); - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_INT, 0L)).isEqualTo(10); - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_INT, 4L)).isEqualTo(777); - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_INT, 8L)).isEqualTo(30); - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_INT, 12L)).isEqualTo(999); - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_INT, 16L)).isEqualTo(50); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_INT, 0L)).isEqualTo(10); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_INT, 4L)).isEqualTo(777); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_INT, 8L)).isEqualTo(30); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_INT, 12L)).isEqualTo(999); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_INT, 16L)).isEqualTo(50); } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DecimalEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DecimalEncodingEncoderTest.java index 6e3306db..9e360fba 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DecimalEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DecimalEncodingEncoderTest.java @@ -5,7 +5,7 @@ import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.decode.DecodeContext; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; import io.github.dfa1.vortex.encoding.TestSegments; @@ -42,7 +42,7 @@ void roundTrip_i64Precision_preservesBuffer() { // Then assertThat(result.length()).isEqualTo(values.length); for (int i = 0; i < values.length; i++) { - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_LONG, (long) i * 8)).isEqualTo(values[i]); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_LONG, (long) i * 8)).isEqualTo(values[i]); } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoderTest.java index 05e88c82..2d276e91 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DeltaEncodingEncoderTest.java @@ -6,7 +6,7 @@ import io.github.dfa1.vortex.encoding.DTypes; import io.github.dfa1.vortex.reader.decode.DecodeContext; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; import io.github.dfa1.vortex.core.proto.ProtoDeltaMetadata; @@ -77,7 +77,7 @@ void encodeDecode_i64_isLossless(long[] data) { // Then assertThat(result.length()).isEqualTo(data.length); for (int i = 0; i < data.length; i++) { - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_LONG, (long) i * 8)).as("index %d", i).isEqualTo(data[i]); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_LONG, (long) i * 8)).as("index %d", i).isEqualTo(data[i]); } } @@ -94,7 +94,7 @@ void encodeDecode_i32_isLossless(int[] data) { // Then assertThat(result.length()).isEqualTo(data.length); for (int i = 0; i < data.length; i++) { - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_INT, (long) i * 4)).as("index %d", i).isEqualTo(data[i]); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_INT, (long) i * 4)).as("index %d", i).isEqualTo(data[i]); } } @@ -111,7 +111,7 @@ void encodeDecode_monotoneI64_isLossless(String name, long[] data) { // Then assertThat(result.length()).isEqualTo(data.length); for (int i = 0; i < data.length; i++) { - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_LONG, (long) i * 8)).as("index %d", i).isEqualTo(data[i]); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_LONG, (long) i * 8)).as("index %d", i).isEqualTo(data[i]); } } @@ -151,9 +151,9 @@ void encodeDecode_randomAcrossPtypesAndSizes_isLossless(String name, DType dtype long off = (long) i * ptype.byteSize(); switch (ptype) { case I8, U8 -> assertThat(seg.get(ValueLayout.JAVA_BYTE, off)).as("idx %d", i).isEqualTo(((byte[]) data)[i]); - case I16, U16 -> assertThat(seg.get(PTypeIO.LE_SHORT, off)).as("idx %d", i).isEqualTo(((short[]) data)[i]); - case I32, U32 -> assertThat(seg.get(PTypeIO.LE_INT, off)).as("idx %d", i).isEqualTo(((int[]) data)[i]); - case I64, U64 -> assertThat(seg.get(PTypeIO.LE_LONG, off)).as("idx %d", i).isEqualTo(((long[]) data)[i]); + case I16, U16 -> assertThat(seg.get(VortexFormat.LE_SHORT, off)).as("idx %d", i).isEqualTo(((short[]) data)[i]); + case I32, U32 -> assertThat(seg.get(VortexFormat.LE_INT, off)).as("idx %d", i).isEqualTo(((int[]) data)[i]); + case I64, U64 -> assertThat(seg.get(VortexFormat.LE_LONG, off)).as("idx %d", i).isEqualTo(((long[]) data)[i]); default -> throw new AssertionError(ptype); } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoderTest.java index 817a36c5..415c19ad 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/DictEncodingEncoderTest.java @@ -5,7 +5,7 @@ import io.github.dfa1.vortex.encoding.DTypes; import io.github.dfa1.vortex.reader.decode.DecodeContext; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; import io.github.dfa1.vortex.core.proto.ProtoDictMetadata; @@ -96,7 +96,7 @@ void encodeDecode_i32_isLossless(int[] data) { // Then assertThat(result.length()).isEqualTo(data.length); for (int i = 0; i < data.length; i++) { - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_INT, (long) i * 4)).as("index %d", i).isEqualTo(data[i]); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_INT, (long) i * 4)).as("index %d", i).isEqualTo(data[i]); } } @@ -113,7 +113,7 @@ void encodeDecode_i64_isLossless(long[] data) { // Then assertThat(result.length()).isEqualTo(data.length); for (int i = 0; i < data.length; i++) { - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_LONG, (long) i * 8)).as("index %d", i).isEqualTo(data[i]); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_LONG, (long) i * 8)).as("index %d", i).isEqualTo(data[i]); } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java index 011d6721..d74a6526 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/FsstEncodingEncoderTest.java @@ -8,7 +8,7 @@ import io.github.dfa1.vortex.reader.decode.DecodeContext; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; import io.github.dfa1.vortex.core.proto.ProtoFSSTMetadata; @@ -129,7 +129,7 @@ private static DecodeContext buildCtx( MemorySegment symBuf = arena.allocate(Math.max(symbols.length * 8L, 1), 8); for (int i = 0; i < symbols.length; i++) { - symBuf.setAtIndex(PTypeIO.LE_LONG, i, symbols[i]); + symBuf.setAtIndex(VortexFormat.LE_LONG, i, symbols[i]); } MemorySegment symLenBuf = arena.allocate(Math.max(symLens.length, 1)); @@ -144,12 +144,12 @@ private static DecodeContext buildCtx( MemorySegment uncompLenBuf = arena.allocate((long) uncompLens.length * Integer.BYTES, Integer.BYTES); for (int i = 0; i < uncompLens.length; i++) { - uncompLenBuf.setAtIndex(PTypeIO.LE_INT, i, uncompLens[i]); + uncompLenBuf.setAtIndex(VortexFormat.LE_INT, i, uncompLens[i]); } MemorySegment codesOffBuf = arena.allocate((long) codesOffsets.length * Integer.BYTES, Integer.BYTES); for (int i = 0; i < codesOffsets.length; i++) { - codesOffBuf.setAtIndex(PTypeIO.LE_INT, i, codesOffsets[i]); + codesOffBuf.setAtIndex(VortexFormat.LE_INT, i, codesOffsets[i]); } MemorySegment[] segs = {symBuf, symLenBuf, compBuf, uncompLenBuf, codesOffBuf}; diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoderTest.java index b0f158d7..9cf3aed6 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoderTest.java @@ -8,7 +8,7 @@ import io.github.dfa1.vortex.reader.array.MaskedArray; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; import io.github.dfa1.vortex.reader.decode.BoolEncodingDecoder; @@ -116,9 +116,9 @@ void inner_containsChildValues() { IntArray inner = (IntArray) result.inner(); // Then - assertThat(inner.materialize(Arena.ofAuto()).get(PTypeIO.LE_INT, 0L)).isEqualTo(7); - assertThat(inner.materialize(Arena.ofAuto()).get(PTypeIO.LE_INT, 4L)).isEqualTo(8); - assertThat(inner.materialize(Arena.ofAuto()).get(PTypeIO.LE_INT, 8L)).isEqualTo(9); + assertThat(inner.materialize(Arena.ofAuto()).get(VortexFormat.LE_INT, 0L)).isEqualTo(7); + assertThat(inner.materialize(Arena.ofAuto()).get(VortexFormat.LE_INT, 4L)).isEqualTo(8); + assertThat(inner.materialize(Arena.ofAuto()).get(VortexFormat.LE_INT, 8L)).isEqualTo(9); } @Test diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PcoEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PcoEncodingEncoderTest.java index 3fc43875..b0be14d1 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PcoEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PcoEncodingEncoderTest.java @@ -1,7 +1,7 @@ package io.github.dfa1.vortex.writer.encode; import io.github.dfa1.vortex.core.model.DType; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.array.Array; import io.github.dfa1.vortex.reader.decode.DecodeContext; @@ -133,7 +133,7 @@ void encodeDecode_i64_isLossless(String name, long[] data) { // Then assertThat(result.length()).isEqualTo(data.length); for (int i = 0; i < data.length; i++) { - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_LONG, (long) i * 8)).as("index %d", i).isEqualTo(data[i]); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_LONG, (long) i * 8)).as("index %d", i).isEqualTo(data[i]); } } @@ -150,7 +150,7 @@ void encodeDecode_u64_isLossless(String name, long[] data) { // Then assertThat(result.length()).isEqualTo(data.length); for (int i = 0; i < data.length; i++) { - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_LONG, (long) i * 8)).as("index %d", i).isEqualTo(data[i]); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_LONG, (long) i * 8)).as("index %d", i).isEqualTo(data[i]); } } @@ -167,7 +167,7 @@ void encodeDecode_i32_isLossless(String name, int[] data) { // Then assertThat(result.length()).isEqualTo(data.length); for (int i = 0; i < data.length; i++) { - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_INT, (long) i * 4)).as("index %d", i).isEqualTo(data[i]); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_INT, (long) i * 4)).as("index %d", i).isEqualTo(data[i]); } } @@ -184,7 +184,7 @@ void encodeDecode_u32_isLossless(String name, int[] data) { // Then assertThat(result.length()).isEqualTo(data.length); for (int i = 0; i < data.length; i++) { - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_INT, (long) i * 4)).as("index %d", i).isEqualTo(data[i]); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_INT, (long) i * 4)).as("index %d", i).isEqualTo(data[i]); } } @@ -201,7 +201,7 @@ void encodeDecode_i16_isLossless(String name, short[] data) { // Then assertThat(result.length()).isEqualTo(data.length); for (int i = 0; i < data.length; i++) { - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_SHORT, (long) i * 2)).as("index %d", i).isEqualTo(data[i]); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_SHORT, (long) i * 2)).as("index %d", i).isEqualTo(data[i]); } } @@ -218,7 +218,7 @@ void encodeDecode_u16_isLossless(String name, short[] data) { // Then assertThat(result.length()).isEqualTo(data.length); for (int i = 0; i < data.length; i++) { - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_SHORT, (long) i * 2)).as("index %d", i).isEqualTo(data[i]); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_SHORT, (long) i * 2)).as("index %d", i).isEqualTo(data[i]); } } @@ -235,7 +235,7 @@ void encodeDecode_f32_isLossless(String name, float[] data) { // Then assertThat(result.length()).isEqualTo(data.length); for (int i = 0; i < data.length; i++) { - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_FLOAT, (long) i * 4)).as("index %d", i).isEqualTo(data[i]); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_FLOAT, (long) i * 4)).as("index %d", i).isEqualTo(data[i]); } } @@ -252,7 +252,7 @@ void encodeDecode_f64_isLossless(String name, double[] data) { // Then assertThat(result.length()).isEqualTo(data.length); for (int i = 0; i < data.length; i++) { - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_DOUBLE, (long) i * 8)).as("index %d", i).isEqualTo(data[i]); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_DOUBLE, (long) i * 8)).as("index %d", i).isEqualTo(data[i]); } } @@ -373,7 +373,7 @@ void encodeDecode_i64_random_isLossless(String name, long[] data) { assertThat(result.length()).isEqualTo(data.length); MemorySegment m = result.materialize(Arena.ofAuto()); for (int i = 0; i < data.length; i++) { - assertThat(m.get(PTypeIO.LE_LONG, (long) i * 8)).as("index %d", i).isEqualTo(data[i]); + assertThat(m.get(VortexFormat.LE_LONG, (long) i * 8)).as("index %d", i).isEqualTo(data[i]); } } @@ -391,7 +391,7 @@ void encodeDecode_i32_random_isLossless(String name, int[] data) { assertThat(result.length()).isEqualTo(data.length); MemorySegment m = result.materialize(Arena.ofAuto()); for (int i = 0; i < data.length; i++) { - assertThat(m.get(PTypeIO.LE_INT, (long) i * 4)).as("index %d", i).isEqualTo(data[i]); + assertThat(m.get(VortexFormat.LE_INT, (long) i * 4)).as("index %d", i).isEqualTo(data[i]); } } @@ -409,7 +409,7 @@ void encodeDecode_f64_random_isLossless(String name, double[] data) { assertThat(result.length()).isEqualTo(data.length); MemorySegment m = result.materialize(Arena.ofAuto()); for (int i = 0; i < data.length; i++) { - assertThat(m.get(PTypeIO.LE_DOUBLE, (long) i * 8)).as("index %d", i).isEqualTo(data[i]); + assertThat(m.get(VortexFormat.LE_DOUBLE, (long) i * 8)).as("index %d", i).isEqualTo(data[i]); } } @@ -427,7 +427,7 @@ void encodeDecode_f32_random_isLossless(String name, float[] data) { assertThat(result.length()).isEqualTo(data.length); MemorySegment m = result.materialize(Arena.ofAuto()); for (int i = 0; i < data.length; i++) { - assertThat(m.get(PTypeIO.LE_FLOAT, (long) i * 4)).as("index %d", i).isEqualTo(data[i]); + assertThat(m.get(VortexFormat.LE_FLOAT, (long) i * 4)).as("index %d", i).isEqualTo(data[i]); } } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoderTest.java index c22d1668..c051a319 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/PrimitiveEncodingEncoderTest.java @@ -8,7 +8,7 @@ import io.github.dfa1.vortex.reader.decode.DecodeContext; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; import io.github.dfa1.vortex.encoding.TestSegments; @@ -78,7 +78,7 @@ void encodeDecode_i64_isLossless(long[] data) { // Then assertThat(result.length()).isEqualTo(data.length); for (int i = 0; i < data.length; i++) { - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_LONG, (long) i * 8)).isEqualTo(data[i]); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_LONG, (long) i * 8)).isEqualTo(data[i]); } } @@ -96,7 +96,7 @@ void encodeDecode_i32_isLossless(int[] data) { // Then assertThat(result.length()).isEqualTo(data.length); for (int i = 0; i < data.length; i++) { - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_INT, (long) i * 4)).isEqualTo(data[i]); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_INT, (long) i * 4)).isEqualTo(data[i]); } } @@ -114,7 +114,7 @@ void encodeDecode_f64_isLossless(double[] data) { // Then assertThat(result.length()).isEqualTo(data.length); for (int i = 0; i < data.length; i++) { - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_DOUBLE, (long) i * 8)).isEqualTo(data[i]); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_DOUBLE, (long) i * 8)).isEqualTo(data[i]); } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/StructEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/StructEncodingEncoderTest.java index 7d77c323..8140c9ed 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/StructEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/StructEncodingEncoderTest.java @@ -10,7 +10,7 @@ import io.github.dfa1.vortex.reader.decode.DecodeContext; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; import io.github.dfa1.vortex.encoding.TestSegments; @@ -141,7 +141,7 @@ void decode_nonNullableWrapper_oneChild_returnsValues() { // Then assertThat(result.length()).isEqualTo(data.length); for (int i = 0; i < data.length; i++) { - assertThat(result.materialize(Arena.ofAuto()).get(PTypeIO.LE_LONG, (long) i * 8)).isEqualTo(data[i]); + assertThat(result.materialize(Arena.ofAuto()).get(VortexFormat.LE_LONG, (long) i * 8)).isEqualTo(data[i]); } } diff --git a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoderTest.java b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoderTest.java index 65fe5807..d0345c73 100644 --- a/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoderTest.java +++ b/writer/src/test/java/io/github/dfa1/vortex/writer/encode/VarBinViewEncodingEncoderTest.java @@ -6,7 +6,7 @@ import io.github.dfa1.vortex.reader.decode.DecodeContext; import io.github.dfa1.vortex.core.model.EncodingId; -import io.github.dfa1.vortex.core.io.PTypeIO; +import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.reader.decode.TestRegistry; import io.github.dfa1.vortex.reader.decode.VarBinViewEncodingDecoder; @@ -133,12 +133,12 @@ void decode_roundtrip_returnsAllStrings(String name, String[] values) { for (int i = 0; i < values.length; i++) { byte[] b = bytesArr[i]; long viewOff = (long) i * 16; - views.set(PTypeIO.LE_INT, viewOff, b.length); + views.set(VortexFormat.LE_INT, viewOff, b.length); if (b.length <= 12) { MemorySegment.copy(MemorySegment.ofArray(b), 0, views, viewOff + 4, b.length); } else { - views.set(PTypeIO.LE_INT, viewOff + 8, 0); - views.set(PTypeIO.LE_INT, viewOff + 12, dataOffset); + views.set(VortexFormat.LE_INT, viewOff + 8, 0); + views.set(VortexFormat.LE_INT, viewOff + 12, dataOffset); MemorySegment.copy(MemorySegment.ofArray(b), 0, dataBuf, dataOffset, b.length); dataOffset += b.length; }