Skip to content

Commit 9f893b8

Browse files
dfa1claude
andcommitted
test(integration): finish OHLC test-data dedup onto OhlcData.Batch
Drop the integration OhlcGenerator/OhlcBatch adapter and switch the JNI round-trip callers (FileSizeComparisonIntegrationTest, JavaWritesRustReadsIntegrationTest) to core.testing.OhlcData.Batch directly, so the OHLC shape is single-sourced rather than mirrored. Verified against the JNI integration suite (OHLC round-trips + file-size comparison, native libs present). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent e82d1d3 commit 9f893b8

4 files changed

Lines changed: 28 additions & 66 deletions

File tree

TODO.md

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,6 @@
66
- [ ] Create website
77
- build something like hardwood.dev but for vortex files
88

9-
## Testing
10-
11-
- [ ] **Finish OHLC test-data dedup** — the random-walk generator is single-sourced in
12-
`core.testing.OhlcData` (core test-jar). `integration`'s `OhlcGenerator` is now a thin adapter
13-
that maps `OhlcData.Batch` to its own `OhlcBatch` (`symbols`/`dates` field names) only to avoid
14-
churning the JNI/Arrow callers. Align fully: drop `OhlcGenerator`/`OhlcBatch`, switch the
15-
integration callers (`FileSizeComparisonIntegrationTest`, `JavaWritesRustReadsIntegrationTest`)
16-
to `OhlcData.Batch` directly so there is one shape, not two. Verify with the JNI integration
17-
suite (needs vortex-jni native libs).
18-
199
## Performance
2010

2111
- [ ] **Benchmark publishing** — drop CI workflow, add `bench-publish` script; see [ADR-0006](docs/adr/0006-benchmark-publishing.md).

integration/src/test/java/io/github/dfa1/vortex/integration/FileSizeComparisonIntegrationTest.java

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import dev.vortex.arrow.ArrowAllocation;
55
import dev.vortex.jni.NativeLoader;
66
import io.github.dfa1.vortex.core.model.DType;
7+
import io.github.dfa1.vortex.core.testing.OhlcData;
78
import io.github.dfa1.vortex.reader.array.LongArray;
89
import io.github.dfa1.vortex.reader.ReadRegistry;
910
import io.github.dfa1.vortex.reader.VortexReader;
@@ -83,13 +84,13 @@ class FileSizeComparisonIntegrationTest {
8384

8485
// ── Writers ───────────────────────────────────────────────────────────────
8586

86-
private static Path writeCsv(Path dir, List<OhlcGenerator.OhlcBatch> batches) throws IOException {
87+
private static Path writeCsv(Path dir, List<OhlcData.Batch> batches) throws IOException {
8788
Path file = dir.resolve("ohlc.csv");
8889
try (BufferedWriter csv = Files.newBufferedWriter(file)) {
8990
csv.write("symbol,date,open,high,low,close,volume\n");
90-
for (OhlcGenerator.OhlcBatch b : batches) {
91-
for (int i = 0; i < b.dates().length; i++) {
92-
csv.write(b.symbols()[i] + "," + LocalDate.ofEpochDay(b.dates()[i]) + ","
91+
for (OhlcData.Batch b : batches) {
92+
for (int i = 0; i < b.date().length; i++) {
93+
csv.write(b.symbol()[i] + "," + LocalDate.ofEpochDay(b.date()[i]) + ","
9394
+ b.open()[i] + "," + b.high()[i] + ","
9495
+ b.low()[i] + "," + b.close()[i] + "," + b.volume()[i] + "\n");
9596
}
@@ -98,41 +99,41 @@ private static Path writeCsv(Path dir, List<OhlcGenerator.OhlcBatch> batches) th
9899
return file;
99100
}
100101

101-
private static Path writeJava(Path dir, List<OhlcGenerator.OhlcBatch> batches) throws IOException {
102+
private static Path writeJava(Path dir, List<OhlcData.Batch> batches) throws IOException {
102103
return writeJava(dir, "ohlc-java.vtx", WriteOptions.cascading(3), batches);
103104
}
104105

105-
private static Path writeJavaZstd(Path dir, List<OhlcGenerator.OhlcBatch> batches) throws IOException {
106+
private static Path writeJavaZstd(Path dir, List<OhlcData.Batch> batches) throws IOException {
106107
return writeJava(dir, "ohlc-java-zstd.vtx", WriteOptions.cascading(3).withZstd(true), batches);
107108
}
108109

109110
private static Path writeJavaGlobalDict(Path dir, boolean globalDict,
110-
List<OhlcGenerator.OhlcBatch> batches) throws IOException {
111+
List<OhlcData.Batch> batches) throws IOException {
111112
String name = globalDict ? "ohlc-java-globaldict.vtx" : "ohlc-java-perchunkdict.vtx";
112113
return writeJava(dir, name, WriteOptions.cascading(3).withGlobalDict(globalDict), batches);
113114
}
114115

115116
private static Path writeJava(Path dir, String filename, WriteOptions opts,
116-
List<OhlcGenerator.OhlcBatch> batches) throws IOException {
117+
List<OhlcData.Batch> batches) throws IOException {
117118
Path file = dir.resolve(filename);
118119
try (FileChannel ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
119120
VortexWriter writer = VortexWriter.create(ch, JAVA_SCHEMA, opts)) {
120-
for (OhlcGenerator.OhlcBatch b : batches) {
121+
for (OhlcData.Batch b : batches) {
121122
writer.writeChunk(Map.of(
122-
"symbol", b.symbols(), "date", b.dates(),
123+
"symbol", b.symbol(), "date", b.date(),
123124
"open", b.open(), "high", b.high(),
124125
"low", b.low(), "close", b.close(), "volume", b.volume()));
125126
}
126127
}
127128
return file;
128129
}
129130

130-
private static Path writeJni(Path dir, List<OhlcGenerator.OhlcBatch> batches) throws IOException {
131+
private static Path writeJni(Path dir, List<OhlcData.Batch> batches) throws IOException {
131132
Path file = dir.resolve("ohlc-jni.vtx");
132133
String uri = file.toAbsolutePath().toUri().toString();
133134
try (dev.vortex.api.VortexWriter writer = dev.vortex.api.VortexWriter.create(
134135
SESSION, uri, JNI_SCHEMA, new HashMap<>(), ALLOCATOR)) {
135-
for (OhlcGenerator.OhlcBatch b : batches) {
136+
for (OhlcData.Batch b : batches) {
136137
try (VectorSchemaRoot root = VectorSchemaRoot.create(JNI_SCHEMA, ALLOCATOR)) {
137138
VarCharVector symbolVec = (VarCharVector) root.getVector("symbol");
138139
DateDayVector dateVec = (DateDayVector) root.getVector("date");
@@ -142,7 +143,7 @@ private static Path writeJni(Path dir, List<OhlcGenerator.OhlcBatch> batches) th
142143
Float8Vector closeVec = (Float8Vector) root.getVector("close");
143144
BigIntVector volVec = (BigIntVector) root.getVector("volume");
144145

145-
int n = b.dates().length;
146+
int n = b.date().length;
146147
symbolVec.allocateNew();
147148
dateVec.allocateNew(n);
148149
openVec.allocateNew(n);
@@ -152,8 +153,8 @@ private static Path writeJni(Path dir, List<OhlcGenerator.OhlcBatch> batches) th
152153
volVec.allocateNew(n);
153154

154155
for (int i = 0; i < n; i++) {
155-
symbolVec.setSafe(i, b.symbols()[i].getBytes(StandardCharsets.UTF_8));
156-
dateVec.setSafe(i, b.dates()[i]);
156+
symbolVec.setSafe(i, b.symbol()[i].getBytes(StandardCharsets.UTF_8));
157+
dateVec.setSafe(i, b.date()[i]);
157158
openVec.setSafe(i, b.open()[i]);
158159
highVec.setSafe(i, b.high()[i]);
159160
lowVec.setSafe(i, b.low()[i]);
@@ -178,7 +179,7 @@ private static Path writeJni(Path dir, List<OhlcGenerator.OhlcBatch> batches) th
178179
@Test
179180
void fileSizeComparison(@TempDir Path tmp) throws IOException {
180181
// Given
181-
List<OhlcGenerator.OhlcBatch> batches = OhlcGenerator.generate(TOTAL_ROWS, BATCH_SIZE);
182+
List<OhlcData.Batch> batches = OhlcData.generate(TOTAL_ROWS, BATCH_SIZE);
182183

183184
// When
184185
Path csvFile = writeCsv(tmp, batches);
@@ -217,7 +218,7 @@ void fileSizeComparison(@TempDir Path tmp) throws IOException {
217218
@Test
218219
void withZstd_smallerFile_and_readable(@TempDir Path tmp) throws IOException {
219220
// Given
220-
List<OhlcGenerator.OhlcBatch> batches = OhlcGenerator.generate(TOTAL_ROWS, BATCH_SIZE);
221+
List<OhlcData.Batch> batches = OhlcData.generate(TOTAL_ROWS, BATCH_SIZE);
221222

222223
// When
223224
Path noZstd = writeJava(tmp, batches);
@@ -246,12 +247,12 @@ void withZstd_smallerFile_and_readable(@TempDir Path tmp) throws IOException {
246247

247248
@Test
248249
void globalDict_multiSymbol_smallerThanPerChunkDict(@TempDir Path tmp) throws IOException {
249-
// Given — multi-symbol OHLC (30 tickers from OhlcGenerator) split across multiple chunks
250+
// Given — multi-symbol OHLC (30 tickers from OhlcData) split across multiple chunks
250251
// so per-chunk-dict mode emits the same ticker dictionary in every chunk while global-dict
251252
// mode emits it once. Small chunkSize amplifies the saving.
252253
int rows = 200_000;
253254
int batch = 20_000;
254-
List<OhlcGenerator.OhlcBatch> batches = OhlcGenerator.generate(rows, batch);
255+
List<OhlcData.Batch> batches = OhlcData.generate(rows, batch);
255256

256257
// When
257258
Path globalDictFile = writeJavaGlobalDict(tmp, true, batches);

integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import dev.vortex.jni.NativeLoader;
1111
import io.github.dfa1.vortex.core.model.DType;
1212
import io.github.dfa1.vortex.core.model.PType;
13+
import io.github.dfa1.vortex.core.testing.OhlcData;
1314
import io.github.dfa1.vortex.writer.encode.BoolEncodingEncoder;
1415
import io.github.dfa1.vortex.writer.encode.PcoEncodingEncoder;
1516
import io.github.dfa1.vortex.writer.encode.ByteBoolEncodingEncoder;
@@ -428,14 +429,14 @@ private static void deleteDir(Path dir) throws IOException {
428429
void javaWriter_jniReader_cascading_ohlc(@TempDir Path tmp) throws IOException {
429430
// Given — OHLC data written with cascading(3): exercises ALP→FOR→bitpacked chain
430431
Path file = tmp.resolve("java_cascade_ohlc.vtx");
431-
List<OhlcGenerator.OhlcBatch> batches = OhlcGenerator.generate(10_000, 1_000);
432+
List<OhlcData.Batch> batches = OhlcData.generate(10_000, 1_000);
432433

433434
try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
434435
var sut = VortexWriter.create(ch, OHLC_SCHEMA, WriteOptions.cascading(3))) {
435-
for (OhlcGenerator.OhlcBatch b : batches) {
436+
for (OhlcData.Batch b : batches) {
436437
sut.writeChunk(Map.of(
437-
"date", b.dates(),
438-
"symbol", b.symbols(),
438+
"date", b.date(),
439+
"symbol", b.symbol(),
439440
"open", b.open(),
440441
"high", b.high(),
441442
"low", b.low(),
@@ -708,12 +709,12 @@ void javaWriter_jniReader_monotonic_i64_cascading(@TempDir Path tmp) throws IOEx
708709
void javaWriter_jniReader_cascading_ohlc_columnProjection(@TempDir Path tmp) throws IOException {
709710
// Given
710711
Path file = tmp.resolve("java_cascade_proj.vtx");
711-
List<OhlcGenerator.OhlcBatch> batches = OhlcGenerator.generate(2_000, 1_000);
712+
List<OhlcData.Batch> batches = OhlcData.generate(2_000, 1_000);
712713
try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
713714
var sut = VortexWriter.create(ch, OHLC_SCHEMA, WriteOptions.cascading(3))) {
714-
for (OhlcGenerator.OhlcBatch b : batches) {
715+
for (OhlcData.Batch b : batches) {
715716
sut.writeChunk(Map.of(
716-
"date", b.dates(), "symbol", b.symbols(),
717+
"date", b.date(), "symbol", b.symbol(),
717718
"open", b.open(), "high", b.high(),
718719
"low", b.low(), "close", b.close(), "volume", b.volume()));
719720
}

integration/src/test/java/io/github/dfa1/vortex/integration/OhlcGenerator.java

Lines changed: 0 additions & 30 deletions
This file was deleted.

0 commit comments

Comments
 (0)