Skip to content

Commit 8ab9ec7

Browse files
dfa1claude
dfa1
and
claude
committed
feat(reader): lazy DateTimeParts reassembly via LazyDateTimePartsLongArray
The pre-existing DateTimePartsEncodingDecoder returned a generic GenericArray wrapping the three children (days, seconds, subseconds) but no consumer in the extension-decode path (ExtensionStorage, TimestampExtensionDecoder, DateExtensionDecoder) knew how to reassemble that shape back into the epoch count their accessors expect. The path was effectively dead at scan time — the encoder tests round-tripped the children individually but never reconstructed an epoch. Add LazyDateTimePartsLongArray (record, implements LongArray) that holds the three children plus the precomputed unitsPerDay / unitsPerSecond multipliers and reassembles on demand: getLong(i) = days[i] * unitsPerDay + seconds[i] * unitsPerSecond + subseconds[i] DateTimePartsArrays (package-private) centralises the per-row read so each child can use whichever signed-integer ptype the encoder picked (Byte / Short / Int / Long Array, optionally wrapped in MaskedArray). DateTimePartsEncodingDecoder parses the parent Extension dtype's TimeUnit metadata byte, computes unitsPerSecond = TimeUnit.divisor() (falling back to 1 for the Days unit, whose seconds and subseconds children are zero) and unitsPerDay = 86_400 × unitsPerSecond, then constructs the lazy record. No buffer allocation, no per-row copy. Now the extension-decode pipeline composes correctly: scanning a vortex.datetimeparts-encoded column under a vortex.timestamp extension produces a LongArray of reassembled epoch counts, which feeds into TimestampExtensionDecoder.instant exactly like a Materialized child. Updated DateTimePartsEncodingEncoderTest to assert the reassembled epoch value instead of the (now hidden) per-child structure — the behaviour the encoder is actually guaranteeing. 3 new unit tests in LazyDateTimePartsLongArrayTest cover the millisecond reassembly, widening from narrower child ptypes, and the fold reduction. ./mvnw verify green (13 modules, integration suite 40s). Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
1 parent 11fe0c4 commit 8ab9ec7

5 files changed

Lines changed: 250 additions & 45 deletions

File tree

‎reader/src/main/java/io/github/dfa1/vortex/reader/array/DateTimePartsArrays.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package io.github.dfa1.vortex.reader.array;
2+
3+
import io.github.dfa1.vortex.core.VortexException;
4+
5+
/// Package-private helper for the {@link LazyDateTimePartsLongArray} record.
6+
///
7+
/// {@code days}, {@code seconds} and {@code subseconds} children can each be one of
8+
/// the four signed-integer typed array interfaces; the writer picks the narrowest
9+
/// ptype that fits. {@link #readLong(Array, long)} centralises the per-row read so
10+
/// the record itself stays compact.
11+
final class DateTimePartsArrays {
12+
13+
private DateTimePartsArrays() {
14+
}
15+
16+
/// Reads {@code arr[i]} as a signed long. Recurses through {@link MaskedArray};
17+
/// throws on null cells so callers don't silently get garbage for nullable
18+
/// columns.
19+
///
20+
/// @param arr source typed Array
21+
/// @param i row index
22+
/// @return cell value as long
23+
/// @throws VortexException for null cells or unsupported array types
24+
static long readLong(Array arr, long i) {
25+
return switch (arr) {
26+
case ByteArray a -> a.getByte(i);
27+
case ShortArray a -> a.getShort(i);
28+
case IntArray a -> a.getInt(i);
29+
case LongArray a -> a.getLong(i);
30+
case MaskedArray a -> {
31+
if (!a.isValid(i)) {
32+
throw new VortexException("DateTimeParts: null cell at index " + i);
33+
}
34+
yield readLong(a.inner(), i);
35+
}
36+
default -> throw new VortexException(
37+
"DateTimeParts: unsupported child array type: " + arr.getClass().getSimpleName());
38+
};
39+
}
40+
}

‎reader/src/main/java/io/github/dfa1/vortex/reader/array/LazyDateTimePartsLongArray.java‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
package io.github.dfa1.vortex.reader.array;
2+
3+
import io.github.dfa1.vortex.core.DType;
4+
5+
import java.util.function.LongBinaryOperator;
6+
import java.util.function.LongConsumer;
7+
8+
/// Lazy {@code vortex.datetimeparts} reassembly as a {@link LongArray}.
9+
///
10+
/// The encoding splits each raw epoch count into three children — {@code days},
11+
/// {@code seconds} (within the day) and {@code subseconds} (within the second).
12+
/// Reconstruction is
13+
///
14+
/// ```
15+
/// raw = days * unitsPerDay + seconds * unitsPerSecond + subseconds
16+
/// ```
17+
///
18+
/// where {@code unitsPerSecond} = `TimeUnit.divisor()` and
19+
/// {@code unitsPerDay} = `86_400 * unitsPerSecond`. The reassembled long carries the
20+
/// same epoch count the downstream extension decoder
21+
/// ({@code TimestampExtensionDecoder}, {@code DateExtensionDecoder}, etc.) expects;
22+
/// no buffer materialisation occurs at construction time.
23+
///
24+
/// The record's {@link #dtype()} is the parent Extension dtype (e.g.
25+
/// {@code vortex.timestamp}) so it slots transparently into the extension-decode
26+
/// pipeline. Children may be any signed integer typed Array
27+
/// ({@link ByteArray}/{@link ShortArray}/{@link IntArray}/{@link LongArray}); the
28+
/// per-row {@link DateTimePartsArrays#readLong} switch handles widening.
29+
///
30+
/// @param dtype logical element type (typically a {@code DType.Extension})
31+
/// @param length total logical row count
32+
/// @param daysArr per-row signed days
33+
/// @param secondsArr per-row signed seconds within the day
34+
/// @param subsecondsArr per-row signed sub-second count
35+
/// @param unitsPerDay multiplier for the days component (= 86_400 × unitsPerSecond)
36+
/// @param unitsPerSecond multiplier for the seconds component (= unit divisor)
37+
public record LazyDateTimePartsLongArray(
38+
DType dtype, long length,
39+
Array daysArr, Array secondsArr, Array subsecondsArr,
40+
long unitsPerDay, long unitsPerSecond)
41+
implements LongArray {
42+
43+
@Override
44+
public long getLong(long i) {
45+
return DateTimePartsArrays.readLong(daysArr, i) * unitsPerDay
46+
+ DateTimePartsArrays.readLong(secondsArr, i) * unitsPerSecond
47+
+ DateTimePartsArrays.readLong(subsecondsArr, i);
48+
}
49+
50+
@Override
51+
public void forEachLong(LongConsumer c) {
52+
long n = length;
53+
for (long i = 0; i < n; i++) {
54+
c.accept(getLong(i));
55+
}
56+
}
57+
58+
@Override
59+
public long fold(long identity, LongBinaryOperator op) {
60+
long acc = identity;
61+
long n = length;
62+
for (long i = 0; i < n; i++) {
63+
acc = op.applyAsLong(acc, getLong(i));
64+
}
65+
return acc;
66+
}
67+
}

‎reader/src/main/java/io/github/dfa1/vortex/reader/decode/DateTimePartsEncodingDecoder.java‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,27 @@
33
import io.github.dfa1.vortex.core.DType;
44
import io.github.dfa1.vortex.core.PType;
55
import io.github.dfa1.vortex.core.VortexException;
6-
import io.github.dfa1.vortex.reader.array.Array;
7-
import io.github.dfa1.vortex.reader.array.GenericArray;
86
import io.github.dfa1.vortex.encoding.EncodingId;
7+
import io.github.dfa1.vortex.encoding.TimeUnit;
98
import io.github.dfa1.vortex.proto.DateTimePartsMetadata;
9+
import io.github.dfa1.vortex.reader.array.Array;
10+
import io.github.dfa1.vortex.reader.array.LazyDateTimePartsLongArray;
1011

1112
import java.io.IOException;
1213
import java.lang.foreign.MemorySegment;
1314
import java.nio.ByteBuffer;
1415

1516
/// Read-only decoder for {@code vortex.datetimeparts}.
17+
///
18+
/// Reassembles the three children (days, seconds, subseconds) into a
19+
/// {@link LazyDateTimePartsLongArray} of epoch counts in the extension's
20+
/// {@link TimeUnit}. No per-row materialisation happens at decode time —
21+
/// the downstream extension decoder reads the reassembled long via the
22+
/// lazy {@code getLong} accessor.
1623
public final class DateTimePartsEncodingDecoder implements EncodingDecoder {
1724

25+
private static final long SECONDS_PER_DAY = 86_400L;
26+
1827
/// Public no-arg constructor required by {@link java.util.ServiceLoader}.
1928
public DateTimePartsEncodingDecoder() {
2029
}
@@ -52,7 +61,27 @@ public Array decode(DecodeContext ctx) {
5261
Array seconds = ctx.decodeChild(1, new DType.Primitive(secondsPtype, false), ctx.rowCount());
5362
Array subseconds = ctx.decodeChild(2, new DType.Primitive(subsecondsPtype, false), ctx.rowCount());
5463

55-
return new GenericArray(ctx.dtype(), ctx.rowCount(), new MemorySegment[0],
56-
new Array[]{days, seconds, subseconds});
64+
if (!(ctx.dtype() instanceof DType.Extension ext)) {
65+
throw new VortexException(EncodingId.VORTEX_DATETIMEPARTS,
66+
"expected Extension dtype, got " + ctx.dtype());
67+
}
68+
long unitsPerSecond = readUnitsPerSecond(ext);
69+
long unitsPerDay = SECONDS_PER_DAY * unitsPerSecond;
70+
71+
return new LazyDateTimePartsLongArray(ctx.dtype(), ctx.rowCount(),
72+
days, seconds, subseconds, unitsPerDay, unitsPerSecond);
73+
}
74+
75+
/// Returns {@code TimeUnit.divisor()} for the extension's declared time unit, or
76+
/// {@code 1} when the unit is {@link TimeUnit#Days} (days carry no sub-second
77+
/// component; seconds and subseconds children are expected to be zero).
78+
private static long readUnitsPerSecond(DType.Extension ext) {
79+
ByteBuffer extMeta = ext.metadata();
80+
if (extMeta == null || !extMeta.hasRemaining()) {
81+
throw new VortexException(EncodingId.VORTEX_DATETIMEPARTS,
82+
"extension " + ext.extensionId() + " missing TimeUnit metadata byte");
83+
}
84+
TimeUnit unit = TimeUnit.fromTag(extMeta.get(extMeta.position()));
85+
return unit == TimeUnit.Days ? 1L : unit.divisor();
5786
}
5887
}

‎reader/src/test/java/io/github/dfa1/vortex/reader/array/LazyDateTimePartsLongArrayTest.java‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
package io.github.dfa1.vortex.reader.array;
2+
3+
import io.github.dfa1.vortex.core.DType;
4+
import io.github.dfa1.vortex.core.PType;
5+
import org.junit.jupiter.api.Test;
6+
7+
import java.lang.foreign.Arena;
8+
import java.lang.foreign.MemorySegment;
9+
import java.lang.foreign.ValueLayout;
10+
11+
import static org.assertj.core.api.Assertions.assertThat;
12+
13+
/// Unit tests for {@link LazyDateTimePartsLongArray}. Verifies the
14+
/// {@code days * unitsPerDay + seconds * unitsPerSecond + subseconds}
15+
/// reassembly across the supported time units, and the widening read path
16+
/// that lets each child use whatever signed-integer ptype the encoder picked.
17+
class LazyDateTimePartsLongArrayTest {
18+
19+
private static final DType I64 = new DType.Primitive(PType.I64, false);
20+
private static final DType I32 = new DType.Primitive(PType.I32, false);
21+
// Decoder constructs records carrying the parent Extension dtype; use I64 here
22+
// as a stand-in since the record never inspects dtype semantics directly.
23+
24+
@Test
25+
void millisecondsReassembly() {
26+
// Given 2 rows of arbitrary (days, seconds_in_day, subseconds) for ms unit.
27+
// unitsPerSecond = 1_000; unitsPerDay = 86_400_000.
28+
// Row 0: days=20_000 -> 2024-12-31 area, seconds=12345, subseconds=678 -> 1_728_012_345_678
29+
try (Arena arena = Arena.ofConfined()) {
30+
LongArray days = longArray(arena, 20_000L, 0L);
31+
LongArray seconds = longArray(arena, 12_345L, 0L);
32+
LongArray subseconds = longArray(arena, 678L, 0L);
33+
long unitsPerSecond = 1_000L;
34+
long unitsPerDay = 86_400L * unitsPerSecond;
35+
36+
var sut = new LazyDateTimePartsLongArray(I64, 2,
37+
days, seconds, subseconds, unitsPerDay, unitsPerSecond);
38+
39+
assertThat(sut.getLong(0)).isEqualTo(
40+
20_000L * unitsPerDay + 12_345L * unitsPerSecond + 678L);
41+
assertThat(sut.getLong(1)).isZero();
42+
}
43+
}
44+
45+
@Test
46+
void widensFromNarrowerChildPtypes() {
47+
// Days as I32, seconds as I32, subseconds as I64 — encoder is free to pick.
48+
try (Arena arena = Arena.ofConfined()) {
49+
IntArray days = intArray(arena, 1);
50+
IntArray seconds = intArray(arena, 2);
51+
LongArray subseconds = longArray(arena, 3L);
52+
long ups = 1_000_000_000L; // nanos
53+
long upd = 86_400L * ups;
54+
55+
var sut = new LazyDateTimePartsLongArray(I64, 1,
56+
days, seconds, subseconds, upd, ups);
57+
58+
assertThat(sut.getLong(0)).isEqualTo(1L * upd + 2L * ups + 3L);
59+
}
60+
}
61+
62+
@Test
63+
void foldSumsAllRows() {
64+
try (Arena arena = Arena.ofConfined()) {
65+
LongArray days = longArray(arena, 1L, 2L, 3L);
66+
LongArray seconds = longArray(arena, 0L, 0L, 0L);
67+
LongArray subseconds = longArray(arena, 0L, 0L, 0L);
68+
long ups = 1L;
69+
long upd = 86_400L;
70+
71+
var sut = new LazyDateTimePartsLongArray(I64, 3,
72+
days, seconds, subseconds, upd, ups);
73+
74+
long sum = sut.fold(0L, Long::sum);
75+
// 1*86400 + 2*86400 + 3*86400 = 6*86400
76+
assertThat(sum).isEqualTo(6L * upd);
77+
}
78+
}
79+
80+
private static LongArray longArray(Arena arena, long... vs) {
81+
MemorySegment seg = arena.allocate(vs.length * 8L, 8);
82+
for (int i = 0; i < vs.length; i++) {
83+
seg.setAtIndex(ValueLayout.JAVA_LONG, i, vs[i]);
84+
}
85+
return new MaterializedLongArray(I64, vs.length, seg.asReadOnly());
86+
}
87+
88+
private static IntArray intArray(Arena arena, int... vs) {
89+
MemorySegment seg = arena.allocate(vs.length * 4L, 4);
90+
for (int i = 0; i < vs.length; i++) {
91+
seg.setAtIndex(ValueLayout.JAVA_INT, i, vs[i]);
92+
}
93+
return new MaterializedIntArray(I32, vs.length, seg.asReadOnly());
94+
}
95+
}

‎writer/src/test/java/io/github/dfa1/vortex/writer/encode/DateTimePartsEncodingEncoderTest.java‎

Lines changed: 15 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import io.github.dfa1.vortex.core.DType;
44
import io.github.dfa1.vortex.core.PType;
5-
import io.github.dfa1.vortex.reader.array.GenericArray;
65
import io.github.dfa1.vortex.reader.array.LongArray;
76
import io.github.dfa1.vortex.reader.decode.ArrayNode;
87
import io.github.dfa1.vortex.encoding.DTypes;
@@ -104,15 +103,10 @@ void roundTrip_milliseconds_preservesDaysSecondsSubseconds() {
104103
MemorySegment[] bufs = result.buffers().toArray(MemorySegment[]::new);
105104
DecodeContext ctx = new DecodeContext(
106105
toArrayNode(result.rootNode()), EXT_TIMESTAMP_MS, 1, bufs, REGISTRY, Arena.global());
107-
GenericArray decoded = (GenericArray) DECODER.decode(ctx);
106+
LongArray decoded = (LongArray) DECODER.decode(ctx);
108107

109108
assertThat(decoded.length()).isEqualTo(1);
110-
LongArray days = (LongArray) decoded.child(0);
111-
LongArray seconds = (LongArray) decoded.child(1);
112-
LongArray subseconds = (LongArray) decoded.child(2);
113-
assertThat(days.getLong(0)).isEqualTo(1L);
114-
assertThat(seconds.getLong(0)).isEqualTo(3723L);
115-
assertThat(subseconds.getLong(0)).isEqualTo(456L);
109+
assertThat(decoded.getLong(0)).isEqualTo(ts);
116110
}
117111

118112
@Test
@@ -126,14 +120,9 @@ void roundTrip_nanoseconds_preservesSubsecondPrecision() {
126120
MemorySegment[] bufs = result.buffers().toArray(MemorySegment[]::new);
127121
DecodeContext ctx = new DecodeContext(
128122
toArrayNode(result.rootNode()), EXT_TIMESTAMP_NS, 1, bufs, REGISTRY, Arena.global());
129-
GenericArray decoded = (GenericArray) DECODER.decode(ctx);
130-
131-
LongArray days = (LongArray) decoded.child(0);
132-
LongArray seconds = (LongArray) decoded.child(1);
133-
LongArray subseconds = (LongArray) decoded.child(2);
134-
assertThat(days.getLong(0)).isEqualTo(1L);
135-
assertThat(seconds.getLong(0)).isEqualTo(3723L);
136-
assertThat(subseconds.getLong(0)).isEqualTo(456_789_123L);
123+
LongArray decoded = (LongArray) DECODER.decode(ctx);
124+
125+
assertThat(decoded.getLong(0)).isEqualTo(ts);
137126
}
138127

139128
@Test
@@ -145,14 +134,9 @@ void roundTrip_epoch_allZero() {
145134
MemorySegment[] bufs = result.buffers().toArray(MemorySegment[]::new);
146135
DecodeContext ctx = new DecodeContext(
147136
toArrayNode(result.rootNode()), EXT_TIMESTAMP_MS, 1, bufs, REGISTRY, Arena.global());
148-
GenericArray decoded = (GenericArray) DECODER.decode(ctx);
149-
150-
LongArray days = (LongArray) decoded.child(0);
151-
LongArray seconds = (LongArray) decoded.child(1);
152-
LongArray subseconds = (LongArray) decoded.child(2);
153-
assertThat(days.getLong(0)).isEqualTo(0L);
154-
assertThat(seconds.getLong(0)).isEqualTo(0L);
155-
assertThat(subseconds.getLong(0)).isEqualTo(0L);
137+
LongArray decoded = (LongArray) DECODER.decode(ctx);
138+
139+
assertThat(decoded.getLong(0)).isZero();
156140
}
157141

158142
@Test
@@ -165,17 +149,12 @@ void roundTrip_multipleTimestamps_allRowsPreserved() {
165149
MemorySegment[] bufs = result.buffers().toArray(MemorySegment[]::new);
166150
DecodeContext ctx = new DecodeContext(
167151
toArrayNode(result.rootNode()), EXT_TIMESTAMP_MS, 4, bufs, REGISTRY, Arena.global());
168-
GenericArray decoded = (GenericArray) DECODER.decode(ctx);
152+
LongArray decoded = (LongArray) DECODER.decode(ctx);
169153

170154
assertThat(decoded.length()).isEqualTo(4);
171-
LongArray days = (LongArray) decoded.child(0);
172-
assertThat(days.getLong(0)).isEqualTo(0L);
173-
assertThat(days.getLong(1)).isEqualTo(1L);
174-
assertThat(days.getLong(2)).isEqualTo(1L);
175-
assertThat(days.getLong(3)).isEqualTo(1L);
176-
LongArray subseconds = (LongArray) decoded.child(2);
177-
assertThat(subseconds.getLong(2)).isEqualTo(0L);
178-
assertThat(subseconds.getLong(3)).isEqualTo(1L);
155+
for (int i = 0; i < timestamps.length; i++) {
156+
assertThat(decoded.getLong(i)).as("row %d", i).isEqualTo(timestamps[i]);
157+
}
179158
}
180159

181160
@ParameterizedTest
@@ -189,14 +168,9 @@ void roundTrip_allUnits_epochIsZero(TimeUnit unit) {
189168
MemorySegment[] bufs = result.buffers().toArray(MemorySegment[]::new);
190169
DecodeContext ctx = new DecodeContext(
191170
toArrayNode(result.rootNode()), dtype, 1, bufs, REGISTRY, Arena.global());
192-
GenericArray decoded = (GenericArray) DECODER.decode(ctx);
193-
194-
LongArray days = (LongArray) decoded.child(0);
195-
LongArray seconds = (LongArray) decoded.child(1);
196-
LongArray subseconds = (LongArray) decoded.child(2);
197-
assertThat(days.getLong(0)).isZero();
198-
assertThat(seconds.getLong(0)).isZero();
199-
assertThat(subseconds.getLong(0)).isZero();
171+
LongArray decoded = (LongArray) DECODER.decode(ctx);
172+
173+
assertThat(decoded.getLong(0)).isZero();
200174
}
201175

202176
@Test

0 commit comments

Comments
 (0)