Skip to content

Commit 2ed02f6

Browse files
dfa1claude
dfa1
and
claude
committed
feat: add VortexInspector for file structure analysis
Shows registered vs actually-used encodings per column by peeking ArrayNode flatbuffers in flat segments. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 9949ea1 commit 2ed02f6

3 files changed

Lines changed: 399 additions & 0 deletions

File tree

‎integration/src/test/java/io/github/dfa1/vortex/integration/OhlcEncodingInspectionIntegrationTest.java‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
package io.github.dfa1.vortex.integration;
2+
3+
import dev.vortex.api.Session;
4+
import dev.vortex.api.VortexWriter;
5+
import dev.vortex.arrow.ArrowAllocation;
6+
import dev.vortex.jni.NativeLoader;
7+
import io.github.dfa1.vortex.encoding.CodecRegistry;
8+
import io.github.dfa1.vortex.io.VortexInspector;
9+
import io.github.dfa1.vortex.io.VortexReader;
10+
import org.apache.arrow.c.ArrowArray;
11+
import org.apache.arrow.c.ArrowSchema;
12+
import org.apache.arrow.c.Data;
13+
import org.apache.arrow.memory.BufferAllocator;
14+
import org.apache.arrow.vector.BigIntVector;
15+
import org.apache.arrow.vector.DateDayVector;
16+
import org.apache.arrow.vector.Float8Vector;
17+
import org.apache.arrow.vector.VarCharVector;
18+
import org.apache.arrow.vector.VectorSchemaRoot;
19+
import org.apache.arrow.vector.types.DateUnit;
20+
import org.apache.arrow.vector.types.FloatingPointPrecision;
21+
import org.apache.arrow.vector.types.pojo.ArrowType;
22+
import org.apache.arrow.vector.types.pojo.Field;
23+
import org.apache.arrow.vector.types.pojo.Schema;
24+
import org.junit.jupiter.api.Test;
25+
import org.junit.jupiter.api.io.TempDir;
26+
27+
import java.io.IOException;
28+
import java.nio.charset.StandardCharsets;
29+
import java.nio.file.Path;
30+
import java.time.LocalDate;
31+
import java.util.HashMap;
32+
import java.util.List;
33+
import java.util.Random;
34+
35+
import static org.assertj.core.api.Assertions.assertThat;
36+
37+
/// Inspects the OHLC file (same schema and data as JniVsJavaReadBenchmark)
38+
/// to reveal which encodings Rust chose for each column, especially "volume".
39+
class OhlcEncodingInspectionIntegrationTest {
40+
41+
private static final int ROWS = 200_000;
42+
private static final int BATCH_SIZE = 50_000;
43+
private static final Session SESSION = Session.create();
44+
private static final BufferAllocator ALLOCATOR = ArrowAllocation.rootAllocator();
45+
private static final ArrowType F64 = new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE);
46+
private static final Schema OHLC_SCHEMA = new Schema(List.of(
47+
Field.notNullable("date", new ArrowType.Date(DateUnit.DAY)),
48+
Field.notNullable("symbol", ArrowType.Utf8.INSTANCE),
49+
Field.notNullable("open", F64),
50+
Field.notNullable("high", F64),
51+
Field.notNullable("low", F64),
52+
Field.notNullable("close", F64),
53+
Field.notNullable("volume", new ArrowType.Int(64, true))
54+
));
55+
56+
static {
57+
NativeLoader.loadJni();
58+
}
59+
60+
@Test
61+
void inspect_ohlcFile_showsColumnEncodings(@TempDir Path tmp) throws IOException {
62+
// Given
63+
Path file = tmp.resolve("ohlc.vtx");
64+
writeOhlc(file, ROWS, BATCH_SIZE);
65+
66+
// When
67+
String report;
68+
try (VortexReader vf = VortexReader.open(file, CodecRegistry.loadAll())) {
69+
report = VortexInspector.inspect(vf);
70+
}
71+
72+
// Then
73+
System.out.println(report);
74+
assertThat(report).contains("volume");
75+
assertThat(report).contains("Used encodings:");
76+
}
77+
78+
private static double round(double v) {
79+
return Math.round(v * 100.0) / 100.0;
80+
}
81+
82+
private static void writeOhlc(Path file, int totalRows, int batchSize) throws IOException {
83+
String uri = file.toAbsolutePath().toUri().toString();
84+
var rng = new Random(42L);
85+
double px = 100.0;
86+
int day = (int) LocalDate.of(2020, 1, 2).toEpochDay();
87+
88+
try (VortexWriter writer = VortexWriter.create(SESSION, uri, OHLC_SCHEMA, new HashMap<>(), ALLOCATOR)) {
89+
int rowsLeft = totalRows;
90+
while (rowsLeft > 0) {
91+
int n = Math.min(rowsLeft, batchSize);
92+
try (VectorSchemaRoot root = VectorSchemaRoot.create(OHLC_SCHEMA, ALLOCATOR)) {
93+
DateDayVector dateVec = (DateDayVector) root.getVector("date");
94+
VarCharVector symbolVec = (VarCharVector) root.getVector("symbol");
95+
Float8Vector openVec = (Float8Vector) root.getVector("open");
96+
Float8Vector highVec = (Float8Vector) root.getVector("high");
97+
Float8Vector lowVec = (Float8Vector) root.getVector("low");
98+
Float8Vector closeVec = (Float8Vector) root.getVector("close");
99+
BigIntVector volumeVec = (BigIntVector) root.getVector("volume");
100+
101+
dateVec.allocateNew(n);
102+
symbolVec.allocateNew(n);
103+
openVec.allocateNew(n);
104+
highVec.allocateNew(n);
105+
lowVec.allocateNew(n);
106+
closeVec.allocateNew(n);
107+
volumeVec.allocateNew(n);
108+
109+
for (int i = 0; i < n; i++) {
110+
double ret = rng.nextGaussian() * 0.02;
111+
double o = round(px * (1 + ret * 0.3));
112+
double c = round(px * (1 + ret));
113+
double spread = Math.abs(px * rng.nextDouble() * 0.03);
114+
double h = round(Math.max(o, c) + spread);
115+
double l = round(Math.min(o, c) - spread);
116+
dateVec.setSafe(i, day++);
117+
symbolVec.setSafe(i, "ACME".getBytes(StandardCharsets.UTF_8));
118+
openVec.setSafe(i, o);
119+
highVec.setSafe(i, h);
120+
lowVec.setSafe(i, l);
121+
closeVec.setSafe(i, c);
122+
volumeVec.setSafe(i, Math.max(100_000L, Math.round(1_000_000 + rng.nextGaussian() * 200_000)));
123+
px = c;
124+
}
125+
root.setRowCount(n);
126+
127+
try (ArrowArray arr = ArrowArray.allocateNew(ALLOCATOR);
128+
ArrowSchema schema = ArrowSchema.allocateNew(ALLOCATOR)) {
129+
Data.exportVectorSchemaRoot(ALLOCATOR, root, null, arr, schema);
130+
writer.writeBatch(arr.memoryAddress(), schema.memoryAddress());
131+
}
132+
}
133+
rowsLeft -= n;
134+
}
135+
}
136+
}
137+
}

‎integration/src/test/java/io/github/dfa1/vortex/integration/VortexInspectorIntegrationTest.java‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
package io.github.dfa1.vortex.integration;
2+
3+
import dev.vortex.api.Session;
4+
import dev.vortex.api.VortexWriter;
5+
import dev.vortex.arrow.ArrowAllocation;
6+
import dev.vortex.jni.NativeLoader;
7+
import io.github.dfa1.vortex.encoding.CodecRegistry;
8+
import io.github.dfa1.vortex.io.VortexInspector;
9+
import io.github.dfa1.vortex.io.VortexReader;
10+
import org.apache.arrow.c.ArrowArray;
11+
import org.apache.arrow.c.ArrowSchema;
12+
import org.apache.arrow.c.Data;
13+
import org.apache.arrow.memory.BufferAllocator;
14+
import org.apache.arrow.vector.BigIntVector;
15+
import org.apache.arrow.vector.Float8Vector;
16+
import org.apache.arrow.vector.VectorSchemaRoot;
17+
import org.apache.arrow.vector.types.FloatingPointPrecision;
18+
import org.apache.arrow.vector.types.pojo.ArrowType;
19+
import org.apache.arrow.vector.types.pojo.Field;
20+
import org.apache.arrow.vector.types.pojo.Schema;
21+
import org.junit.jupiter.api.Test;
22+
import org.junit.jupiter.api.io.TempDir;
23+
24+
import java.io.IOException;
25+
import java.nio.file.Path;
26+
import java.util.HashMap;
27+
import java.util.List;
28+
import java.util.Random;
29+
30+
import static org.assertj.core.api.Assertions.assertThat;
31+
32+
/// Verifies that VortexInspector produces a correct report for a JNI-written file.
33+
class VortexInspectorIntegrationTest {
34+
35+
private static final Session SESSION = Session.create();
36+
private static final BufferAllocator ALLOCATOR = ArrowAllocation.rootAllocator();
37+
private static final Schema SCHEMA = new Schema(List.of(
38+
Field.notNullable("id", new ArrowType.Int(64, true)),
39+
Field.notNullable("value", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE))
40+
));
41+
42+
static {
43+
NativeLoader.loadJni();
44+
}
45+
46+
@Test
47+
void inspect_showsFileInfoAndEncodings(@TempDir Path tmp) throws IOException {
48+
// Given
49+
Path file = tmp.resolve("inspect.vtx");
50+
writeJni(file, 50_000);
51+
52+
// When
53+
String report;
54+
try (VortexReader vf = VortexReader.open(file, CodecRegistry.loadAll())) {
55+
report = VortexInspector.inspect(vf);
56+
}
57+
58+
// Then
59+
System.out.println(report);
60+
assertThat(report).contains("Vortex v");
61+
assertThat(report).contains("id");
62+
assertThat(report).contains("value");
63+
assertThat(report).contains("Registered encodings:");
64+
assertThat(report).contains("Used encodings:");
65+
assertThat(report).contains("Layout:");
66+
}
67+
68+
private static void writeJni(Path file, int rows) throws IOException {
69+
String uri = file.toAbsolutePath().toUri().toString();
70+
var rng = new Random(42L);
71+
try (VortexWriter writer = VortexWriter.create(SESSION, uri, SCHEMA, new HashMap<>(), ALLOCATOR)) {
72+
try (VectorSchemaRoot root = VectorSchemaRoot.create(SCHEMA, ALLOCATOR)) {
73+
BigIntVector idVec = (BigIntVector) root.getVector("id");
74+
Float8Vector valVec = (Float8Vector) root.getVector("value");
75+
idVec.allocateNew(rows);
76+
valVec.allocateNew(rows);
77+
for (int i = 0; i < rows; i++) {
78+
idVec.setSafe(i, i);
79+
valVec.setSafe(i, rng.nextDouble() * 1000.0);
80+
}
81+
root.setRowCount(rows);
82+
try (ArrowArray arr = ArrowArray.allocateNew(ALLOCATOR);
83+
ArrowSchema schema = ArrowSchema.allocateNew(ALLOCATOR)) {
84+
Data.exportVectorSchemaRoot(ALLOCATOR, root, null, arr, schema);
85+
writer.writeBatch(arr.memoryAddress(), schema.memoryAddress());
86+
}
87+
}
88+
}
89+
}
90+
}

0 commit comments

Comments
 (0)