Skip to content

Commit 9f4f3d9

Browse files
dfa1claude
dfa1
and
claude
committed
feat(encoding): implement vortex.datetimeparts encode
Split timestamps into days/seconds/subseconds using Rust's hand-rolled TimestampOptions wire format (byte[0]=unit tag, bytes[1-2]=tz_len u16 LE). Encoder reads TimeUnit from DType.Extension metadata and produces three I64 PrimitiveEncoding children; Decoder was already complete. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent e3113bf commit 9f4f3d9

5 files changed

Lines changed: 411 additions & 59 deletions

File tree

‎TODO.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@
128128
| `vortex.alprd` | — | ❌ | ❌ | hard | ALP-RD: splits float bit pattern into left (dict-compressed, ≤8 entries, 3-bit indices) + right (bitpacked residuals); split point per-array in metadata; two separately-encoded children; harder than ALP; unblocks `alprd.vortex` |
129129
| `vortex.decimal` | `DecimalEncoding` | ✅ | ✅ | — | Decimal |
130130
| `vortex.decimal_byte_parts` | `DecimalBytePartsEncoding` | ✅ | ✅ | — | Decimal byte parts |
131-
| `vortex.datetimeparts` | `DateTimePartsEncoding` | ✅ | ❌ stub | — | Timestamp parts |
131+
| `vortex.datetimeparts` | `DateTimePartsEncoding` | ✅ | ✅ | — | Timestamp parts |
132132
| `vortex.list` | `ListEncoding` | ✅ | ✅ | — | two children: elements + offsets (I64); `ListArray`; cascadable offsets via `decodeChildAs` |
133133
| `vortex.listview` | `ListViewEncoding` | ✅ | ✅ | — | three children: elements + offsets (len N) + sizes (len N); fixture uses U16 for both |
134134
| `vortex.fixed_size_list` | `FixedSizeListEncoding` | ✅ | ✅ | — | one child: flat elements; no offsets |

‎core/src/main/java/io/github/dfa1/vortex/encoding/DateTimePartsData.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
package io.github.dfa1.vortex.encoding;
2+
3+
/// Input data for {@link DateTimePartsEncoding} encoder.
4+
///
5+
/// @param timestamps raw i64 timestamps (number of time units since Unix epoch)
6+
/// @param nullable whether the array has a validity (null) dimension
7+
public record DateTimePartsData(long[] timestamps, boolean nullable) {
8+
}

‎core/src/main/java/io/github/dfa1/vortex/encoding/DateTimePartsEncoding.java‎

Lines changed: 138 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,15 @@
66
import io.github.dfa1.vortex.core.VortexException;
77
import io.github.dfa1.vortex.core.array.Array;
88
import io.github.dfa1.vortex.core.array.GenericArray;
9+
import io.github.dfa1.vortex.proto.DTypeProtos;
910
import io.github.dfa1.vortex.proto.EncodingProtos;
1011

1112
import java.lang.foreign.MemorySegment;
1213
import java.nio.ByteBuffer;
14+
import java.util.ArrayList;
15+
import java.util.List;
1316

14-
/// Decoder for {@code vortex.datetimeparts} — timestamp split into days/seconds/subseconds.
17+
/// Encoder/decoder for {@code vortex.datetimeparts} — timestamp split into days/seconds/subseconds.
1518
///
1619
/// <p>Wire format (per Rust vtable):
1720
/// <ul>
@@ -25,63 +28,140 @@
2528
/// <li>Slot 2 — {@code subseconds}: {@code Primitive(subseconds_ptype, false)}
2629
/// </ul>
2730
/// </ul>
31+
///
32+
/// <p>Extension metadata for {@code vortex.timestamp} dtype (hand-rolled, not protobuf):
33+
/// {@code byte[0]=TimeUnit tag, bytes[1-2]=tz_len (u16 LE), bytes[3+]=tz UTF-8}
2834
public final class DateTimePartsEncoding implements Encoding {
2935

30-
@Override
31-
public EncodingId encodingId() {
32-
return EncodingId.VORTEX_DATETIMEPARTS;
33-
}
34-
35-
@Override
36-
public boolean accepts(DType dtype) {
37-
return dtype instanceof DType.Extension;
38-
}
39-
40-
@Override
41-
public EncodeResult encode(DType dtype, Object data) {
42-
throw new UnsupportedOperationException("encode not yet implemented for " + encodingId());
43-
}
44-
45-
@Override
46-
public Array decode(DecodeContext ctx) {
47-
return Decoder.decode(ctx);
48-
}
49-
50-
private static final class Decoder {
51-
52-
private static Array decode(DecodeContext ctx) {
53-
ByteBuffer meta = ctx.metadata();
54-
if (meta == null || meta.remaining() == 0) {
55-
throw new VortexException(EncodingId.VORTEX_DATETIMEPARTS, "missing metadata");
56-
}
57-
EncodingProtos.DateTimePartsMetadata decoded;
58-
try {
59-
byte[] bytes = new byte[meta.remaining()];
60-
meta.duplicate().get(bytes);
61-
decoded = EncodingProtos.DateTimePartsMetadata.parseFrom(bytes);
62-
} catch (InvalidProtocolBufferException e) {
63-
throw new VortexException(EncodingId.VORTEX_DATETIMEPARTS, "invalid metadata: " + e.getMessage());
64-
}
65-
66-
PType daysPtype = PType.values()[decoded.getDaysPtypeValue()];
67-
PType secondsPtype = PType.values()[decoded.getSecondsPtypeValue()];
68-
PType subsecondsPtype = PType.values()[decoded.getSubsecondsPtypeValue()];
69-
boolean nullable = ctx.dtype().nullable();
70-
71-
Array days = decodeChild(ctx, 0, new DType.Primitive(daysPtype, nullable));
72-
Array seconds = decodeChild(ctx, 1, new DType.Primitive(secondsPtype, false));
73-
Array subseconds = decodeChild(ctx, 2, new DType.Primitive(subsecondsPtype, false));
74-
75-
return new GenericArray(ctx.dtype(), ctx.rowCount(), new MemorySegment[0],
76-
new Array[]{days, seconds, subseconds});
77-
}
78-
79-
private static Array decodeChild(DecodeContext ctx, int idx, DType childDtype) {
80-
ArrayNode childNode = ctx.node().children()[idx];
81-
DecodeContext childCtx = new DecodeContext(
82-
childNode, childDtype, ctx.rowCount(),
83-
ctx.segmentBuffers(), ctx.registry(), ctx.arena());
84-
return ctx.registry().decode(childCtx);
85-
}
86-
}
36+
@Override
37+
public EncodingId encodingId() {
38+
return EncodingId.VORTEX_DATETIMEPARTS;
39+
}
40+
41+
@Override
42+
public boolean accepts(DType dtype) {
43+
return dtype instanceof DType.Extension;
44+
}
45+
46+
@Override
47+
public EncodeResult encode(DType dtype, Object data) {
48+
return Encoder.encode((DType.Extension) dtype, (DateTimePartsData) data);
49+
}
50+
51+
@Override
52+
public Array decode(DecodeContext ctx) {
53+
return Decoder.decode(ctx);
54+
}
55+
56+
private static final class Encoder {
57+
58+
private static final long SECONDS_PER_DAY = 86_400L;
59+
private static final DType I64 = new DType.Primitive(PType.I64, false);
60+
private static final DType I64_NULLABLE = new DType.Primitive(PType.I64, true);
61+
private static final DTypeProtos.PType I64_PROTO =
62+
DTypeProtos.PType.forNumber(PType.I64.ordinal());
63+
64+
static EncodeResult encode(DType.Extension dtype, DateTimePartsData data) {
65+
ByteBuffer extMeta = dtype.metadata();
66+
if (extMeta == null || extMeta.remaining() < 3) {
67+
throw new VortexException(EncodingId.VORTEX_DATETIMEPARTS,
68+
"extension metadata missing or too short");
69+
}
70+
byte[] extBytes = new byte[extMeta.remaining()];
71+
extMeta.duplicate().get(extBytes);
72+
TimeUnit unit = TimeUnit.fromTag(extBytes[0]);
73+
74+
long divisor = unit.divisor();
75+
long ticksPerDay = SECONDS_PER_DAY * divisor;
76+
int n = data.timestamps().length;
77+
78+
long[] days = new long[n];
79+
long[] seconds = new long[n];
80+
long[] subseconds = new long[n];
81+
82+
for (int i = 0; i < n; i++) {
83+
long ts = data.timestamps()[i];
84+
long d = ts / ticksPerDay;
85+
long rem = ts % ticksPerDay;
86+
if (rem < 0) {
87+
rem += ticksPerDay;
88+
d--;
89+
}
90+
days[i] = d;
91+
seconds[i] = rem / divisor;
92+
subseconds[i] = rem % divisor;
93+
}
94+
95+
PrimitiveEncoding primEnc = new PrimitiveEncoding();
96+
DType daysDtype = data.nullable() ? I64_NULLABLE : I64;
97+
98+
EncodeResult daysResult = primEnc.encode(daysDtype, days);
99+
EncodeResult secondsResult = primEnc.encode(I64, seconds);
100+
EncodeResult subsecondsResult = primEnc.encode(I64, subseconds);
101+
102+
List<MemorySegment> allBuffers = new ArrayList<>();
103+
allBuffers.addAll(daysResult.buffers());
104+
allBuffers.addAll(secondsResult.buffers());
105+
allBuffers.addAll(subsecondsResult.buffers());
106+
107+
int off1 = daysResult.buffers().size();
108+
int off2 = off1 + secondsResult.buffers().size();
109+
110+
EncodeNode daysNode = EncodeNode.remapBufferIndices(daysResult.rootNode(), 0);
111+
EncodeNode secondsNode = EncodeNode.remapBufferIndices(secondsResult.rootNode(), off1);
112+
EncodeNode subsecondsNode = EncodeNode.remapBufferIndices(subsecondsResult.rootNode(), off2);
113+
114+
byte[] metaBytes = EncodingProtos.DateTimePartsMetadata.newBuilder()
115+
.setDaysPtype(I64_PROTO)
116+
.setSecondsPtype(I64_PROTO)
117+
.setSubsecondsPtype(I64_PROTO)
118+
.build()
119+
.toByteArray();
120+
121+
EncodeNode root = new EncodeNode(
122+
EncodingId.VORTEX_DATETIMEPARTS,
123+
ByteBuffer.wrap(metaBytes),
124+
new EncodeNode[]{daysNode, secondsNode, subsecondsNode},
125+
new int[]{});
126+
return new EncodeResult(root, List.copyOf(allBuffers), null, null);
127+
}
128+
}
129+
130+
private static final class Decoder {
131+
132+
private static Array decode(DecodeContext ctx) {
133+
ByteBuffer meta = ctx.metadata();
134+
if (meta == null || meta.remaining() == 0) {
135+
throw new VortexException(EncodingId.VORTEX_DATETIMEPARTS, "missing metadata");
136+
}
137+
EncodingProtos.DateTimePartsMetadata decoded;
138+
try {
139+
byte[] bytes = new byte[meta.remaining()];
140+
meta.duplicate().get(bytes);
141+
decoded = EncodingProtos.DateTimePartsMetadata.parseFrom(bytes);
142+
} catch (InvalidProtocolBufferException e) {
143+
throw new VortexException(EncodingId.VORTEX_DATETIMEPARTS, "invalid metadata: " + e.getMessage());
144+
}
145+
146+
PType daysPtype = PType.values()[decoded.getDaysPtypeValue()];
147+
PType secondsPtype = PType.values()[decoded.getSecondsPtypeValue()];
148+
PType subsecondsPtype = PType.values()[decoded.getSubsecondsPtypeValue()];
149+
boolean nullable = ctx.dtype().nullable();
150+
151+
Array days = decodeChild(ctx, 0, new DType.Primitive(daysPtype, nullable));
152+
Array seconds = decodeChild(ctx, 1, new DType.Primitive(secondsPtype, false));
153+
Array subseconds = decodeChild(ctx, 2, new DType.Primitive(subsecondsPtype, false));
154+
155+
return new GenericArray(ctx.dtype(), ctx.rowCount(), new MemorySegment[0],
156+
new Array[]{days, seconds, subseconds});
157+
}
158+
159+
private static Array decodeChild(DecodeContext ctx, int idx, DType childDtype) {
160+
ArrayNode childNode = ctx.node().children()[idx];
161+
DecodeContext childCtx = new DecodeContext(
162+
childNode, childDtype, ctx.rowCount(),
163+
ctx.segmentBuffers(), ctx.registry(), ctx.arena());
164+
return ctx.registry().decode(childCtx);
165+
}
166+
}
87167
}

‎core/src/main/java/io/github/dfa1/vortex/encoding/TimeUnit.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package io.github.dfa1.vortex.encoding;
2+
3+
/// Time unit for timestamp values. Ordinals match Rust's {@code TimeUnit} enum.
4+
public enum TimeUnit {
5+
Nanoseconds, // 0
6+
Microseconds, // 1
7+
Milliseconds, // 2
8+
Seconds, // 3
9+
Days; // 4
10+
11+
public static TimeUnit fromTag(byte tag) {
12+
int i = Byte.toUnsignedInt(tag);
13+
TimeUnit[] values = values();
14+
if (i >= values.length) {
15+
throw new IllegalArgumentException("unknown TimeUnit tag: " + i);
16+
}
17+
return values[i];
18+
}
19+
20+
public long divisor() {
21+
return switch (this) {
22+
case Nanoseconds -> 1_000_000_000L;
23+
case Microseconds -> 1_000_000L;
24+
case Milliseconds -> 1_000L;
25+
case Seconds -> 1L;
26+
case Days -> throw new IllegalArgumentException("Days cannot be split");
27+
};
28+
}
29+
}

0 commit comments

Comments
 (0)