Skip to content

Commit 4d8e25b

Browse files
dfa1claude
dfa1
and
claude
committed
test(integration): expand JavaWritesRustReadsIntegrationTest with 5 more tests
Add: I32 column, Utf8/VarBin column, large chunk (2048 rows = 2 FastLanes blocks), monotonic I64 with cascading, cascaded OHLC column projection. Uncovered: DictEncoding.encodeUtf8 writes non-protobuf metadata — Rust reader rejects it. Utf8 test forces VarBinEncoding explicitly as workaround; bug documented in TODO.md. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 0e12a45 commit 4d8e25b

2 files changed

Lines changed: 167 additions & 0 deletions

File tree

‎TODO.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@
33
## Encodings
44

55
- [ ] the classes are very long of complex, most likely we should group the impl detail in private static inner classes Encoder/Decoder
6+
- [ ] **`DictEncoding.encodeUtf8` Rust cross-compat bug**: `encodeUtf8` writes a 2-byte custom metadata
7+
format (`[codePType ordinal, I64 ordinal]`) instead of a protobuf `DictMetadata` message. Rust reader
8+
calls `parseFrom` on these 2 bytes and fails with "invalid tag value: 0" when codePType = U8 (ordinal 0).
9+
Fix: rewrite `encodeUtf8` metadata to emit a proper `DictLayoutMetadata` protobuf (matching Rust wire format).
610

711
## Performance
812

‎integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,14 @@
1010
import dev.vortex.jni.NativeLoader;
1111
import io.github.dfa1.vortex.core.DType;
1212
import io.github.dfa1.vortex.core.PType;
13+
import io.github.dfa1.vortex.encoding.VarBinEncoding;
1314
import io.github.dfa1.vortex.writer.VortexWriter;
1415
import io.github.dfa1.vortex.writer.WriteOptions;
1516
import org.apache.arrow.memory.BufferAllocator;
1617
import org.apache.arrow.vector.BigIntVector;
1718
import org.apache.arrow.vector.Float8Vector;
19+
import org.apache.arrow.vector.IntVector;
20+
import org.apache.arrow.vector.VarCharVector;
1821
import org.apache.arrow.vector.VectorSchemaRoot;
1922
import org.apache.arrow.vector.ipc.ArrowReader;
2023
import org.junit.jupiter.api.Test;
@@ -42,6 +45,21 @@ class JavaWritesRustReadsIntegrationTest {
4245
new DType.Primitive(PType.F64, false)),
4346
false);
4447

48+
private static final DType.Struct I32_SCHEMA = new DType.Struct(
49+
List.of("v"),
50+
List.of(new DType.Primitive(PType.I32, false)),
51+
false);
52+
53+
private static final DType.Struct STRING_SCHEMA = new DType.Struct(
54+
List.of("s"),
55+
List.of(new DType.Utf8(false)),
56+
false);
57+
58+
private static final DType.Struct TS_SCHEMA = new DType.Struct(
59+
List.of("ts"),
60+
List.of(new DType.Primitive(PType.I64, false)),
61+
false);
62+
4563
private static final DType.Struct OHLC_SCHEMA = new DType.Struct(
4664
List.of("date", "symbol", "open", "high", "low", "close", "volume"),
4765
List.of(
@@ -104,6 +122,52 @@ private static double[] readDoubleColumn(Path file, String column) throws IOExce
104122
return doubles.stream().mapToDouble(Double::doubleValue).toArray();
105123
}
106124

125+
private static int[] readIntColumn(Path file, String column) throws IOException {
126+
String uri = file.toAbsolutePath().toUri().toString();
127+
ScanOptions opts = ScanOptions.builder()
128+
.projection(Expression.select(new String[]{column}, Expression.root()))
129+
.build();
130+
var ints = new ArrayList<Integer>();
131+
DataSource ds = DataSource.open(SESSION, uri);
132+
Scan scan = ds.scan(opts);
133+
while (scan.hasNext()) {
134+
Partition partition = scan.next();
135+
try (ArrowReader reader = partition.scanArrow(ALLOCATOR)) {
136+
while (reader.loadNextBatch()) {
137+
VectorSchemaRoot root = reader.getVectorSchemaRoot();
138+
IntVector vec = (IntVector) root.getVector(column);
139+
for (int i = 0; i < root.getRowCount(); i++) {
140+
ints.add(vec.get(i));
141+
}
142+
}
143+
}
144+
}
145+
return ints.stream().mapToInt(Integer::intValue).toArray();
146+
}
147+
148+
private static String[] readStringColumn(Path file, String column) throws IOException {
149+
String uri = file.toAbsolutePath().toUri().toString();
150+
ScanOptions opts = ScanOptions.builder()
151+
.projection(Expression.select(new String[]{column}, Expression.root()))
152+
.build();
153+
var strings = new ArrayList<String>();
154+
DataSource ds = DataSource.open(SESSION, uri);
155+
Scan scan = ds.scan(opts);
156+
while (scan.hasNext()) {
157+
Partition partition = scan.next();
158+
try (ArrowReader reader = partition.scanArrow(ALLOCATOR)) {
159+
while (reader.loadNextBatch()) {
160+
VectorSchemaRoot root = reader.getVectorSchemaRoot();
161+
VarCharVector vec = (VarCharVector) root.getVector(column);
162+
for (int i = 0; i < root.getRowCount(); i++) {
163+
strings.add(vec.getObject(i).toString());
164+
}
165+
}
166+
}
167+
}
168+
return strings.toArray(String[]::new);
169+
}
170+
107171
// ── JNI read helpers ──────────────────────────────────────────────────────
108172

109173
@Test
@@ -174,4 +238,103 @@ void javaWriter_jniReader_multipleChunks(@TempDir Path tmp) throws IOException {
174238
// Then — JNI may merge chunks; verify all values present regardless of partition count
175239
assertThat(decodedIds).containsExactly(1L, 2L, 3L, 4L, 5L);
176240
}
241+
242+
@Test
243+
void javaWriter_jniReader_i32Column(@TempDir Path tmp) throws IOException {
244+
// Given
245+
Path file = tmp.resolve("java_i32.vtx");
246+
int[] data = {-100, 0, 1, 127, Integer.MAX_VALUE, Integer.MIN_VALUE};
247+
try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
248+
var sut = VortexWriter.create(ch, I32_SCHEMA, WriteOptions.defaults())) {
249+
// When
250+
sut.writeChunk(Map.of("v", data));
251+
}
252+
253+
// Then
254+
int[] decoded = readIntColumn(file, "v");
255+
assertThat(decoded).containsExactly(data);
256+
}
257+
258+
@Test
259+
void javaWriter_jniReader_utf8Column(@TempDir Path tmp) throws IOException {
260+
// Given — force VarBinEncoding; DictEncoding.encodeUtf8 writes non-protobuf metadata
261+
// that the Rust reader rejects (known issue, tracked in DictEncoding)
262+
Path file = tmp.resolve("java_utf8.vtx");
263+
String[] data = {"apple", "banana", "cherry", "date", "elderberry"};
264+
try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
265+
var sut = VortexWriter.create(ch, STRING_SCHEMA, WriteOptions.defaults(),
266+
List.of(new VarBinEncoding()))) {
267+
// When
268+
sut.writeChunk(Map.of("s", data));
269+
}
270+
271+
// Then
272+
String[] decoded = readStringColumn(file, "s");
273+
assertThat(decoded).containsExactly(data);
274+
}
275+
276+
@Test
277+
void javaWriter_jniReader_largeChunk_twoFastLanesBlocks(@TempDir Path tmp) throws IOException {
278+
// Given — 2048 rows = exactly 2 full 1024-element FastLanes blocks
279+
Path file = tmp.resolve("java_large.vtx");
280+
int n = 2048;
281+
long[] ids = new long[n];
282+
double[] vals = new double[n];
283+
for (int i = 0; i < n; i++) {
284+
ids[i] = i;
285+
vals[i] = i * 0.5;
286+
}
287+
try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
288+
var sut = VortexWriter.create(ch, SCHEMA, WriteOptions.defaults())) {
289+
// When
290+
sut.writeChunk(Map.of("id", ids, "value", vals));
291+
}
292+
293+
// Then
294+
long[] decodedIds = readLongColumn(file, "id");
295+
assertThat(decodedIds).containsExactly(ids);
296+
}
297+
298+
@Test
299+
void javaWriter_jniReader_monotonic_i64_cascading(@TempDir Path tmp) throws IOException {
300+
// Given — monotonic timestamps: FOR reduces to constant delta, bitpacked to ~10 bits
301+
Path file = tmp.resolve("java_ts.vtx");
302+
int n = 5_000;
303+
long[] ts = new long[n];
304+
for (int i = 0; i < n; i++) {
305+
ts[i] = 1_700_000_000L + (long) i * 1_000;
306+
}
307+
try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
308+
var sut = VortexWriter.create(ch, TS_SCHEMA, WriteOptions.cascading(3))) {
309+
// When
310+
sut.writeChunk(Map.of("ts", ts));
311+
}
312+
313+
// Then
314+
long[] decoded = readLongColumn(file, "ts");
315+
assertThat(decoded).containsExactly(ts);
316+
}
317+
318+
@Test
319+
void javaWriter_jniReader_cascading_ohlc_columnProjection(@TempDir Path tmp) throws IOException {
320+
// Given
321+
Path file = tmp.resolve("java_cascade_proj.vtx");
322+
List<OhlcGenerator.OhlcBatch> batches = OhlcGenerator.generate(2_000, 1_000);
323+
try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
324+
var sut = VortexWriter.create(ch, OHLC_SCHEMA, WriteOptions.cascading(3))) {
325+
for (OhlcGenerator.OhlcBatch b : batches) {
326+
sut.writeChunk(Map.of(
327+
"date", b.dates(), "symbol", b.symbols(),
328+
"open", b.open(), "high", b.high(),
329+
"low", b.low(), "close", b.close(), "volume", b.volume()));
330+
}
331+
}
332+
333+
// When — project only volume
334+
long[] volumes = readLongColumn(file, "volume");
335+
336+
// Then
337+
long[] expected = batches.stream().flatMapToLong(b -> Arrays.stream(b.volume())).toArray();
338+
assertThat(volumes).containsExactlyInAnyOrder(expected);
339+
}
177340
}

0 commit comments

Comments
 (0)