Skip to content

Commit 6237e38

Browse files
dfa1claude
andcommitted
feat(perf): add symbol column to write benchmark
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 9edb6d3 commit 6237e38

2 files changed

Lines changed: 50 additions & 30 deletions

File tree

TODO.md

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,6 @@
66

77
## Performance
88

9-
- [ ] **#10b Java vs JNI write benchmark** (`performance/` module, `-Pperformance`)
10-
- Add `RustVsJavaWriteBenchmark` mirroring read side: same 10M-row OHLC fixture, JMH throughput, both writers.
11-
- Old `WriteBenchmark.java` (Java-only) removed; rewrite from scratch using JNI bindings already on classpath
12-
(`dev.vortex:vortex-jni:0.72.0`).
13-
149
- [ ] **#10c Publish reproducible perf artifacts**
1510
- Capture JMH JSON + JFR profile alongside README table; cite hardware (CPU model), JDK build (`java -version`),
1611
and benchmark commit SHA so numbers don't rot silently.

performance/src/main/java/io/github/dfa1/vortex/performance/RustVsJavaWriteBenchmark.java

Lines changed: 50 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import org.apache.arrow.vector.DateDayVector;
1515
import org.apache.arrow.vector.Float8Vector;
1616
import org.apache.arrow.vector.BigIntVector;
17+
import org.apache.arrow.vector.VarCharVector;
1718
import org.apache.arrow.vector.VectorSchemaRoot;
1819
import org.apache.arrow.vector.types.DateUnit;
1920
import org.apache.arrow.vector.types.FloatingPointPrecision;
@@ -35,6 +36,7 @@
3536

3637
import java.io.IOException;
3738
import java.nio.channels.FileChannel;
39+
import java.nio.charset.StandardCharsets;
3840
import java.nio.file.Files;
3941
import java.nio.file.Path;
4042
import java.nio.file.StandardOpenOption;
@@ -46,10 +48,9 @@
4648
import java.util.Random;
4749
import java.util.concurrent.TimeUnit;
4850

49-
/// Write benchmark: Java writer vs JNI (Rust) writer on the same numeric OHLC dataset.
51+
/// Write benchmark: Java writer vs JNI (Rust) writer on the same OHLC dataset.
5052
///
51-
/// Schema: date(I32/DATE32), open/high/low/close(F64), volume(I64) — 6 columns, no strings.
52-
/// String column (symbol) omitted: Java writer does not yet support varbin.
53+
/// Schema: date(I32/DATE32), symbol(Utf8), open/high/low/close(F64), volume(I64) — 7 columns.
5354
///
5455
/// Data is pre-generated in @Setup so only encoding + I/O is measured.
5556
/// Each invocation writes 10 M rows; the file size is returned as the benchmark result
@@ -75,6 +76,7 @@ public class RustVsJavaWriteBenchmark {
7576
private static final ArrowType F64_TYPE = new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE);
7677
private static final Schema JNI_SCHEMA = new Schema(List.of(
7778
Field.notNullable("date", new ArrowType.Date(DateUnit.DAY)),
79+
Field.notNullable("symbol", ArrowType.Utf8.INSTANCE),
7880
Field.notNullable("open", F64_TYPE),
7981
Field.notNullable("high", F64_TYPE),
8082
Field.notNullable("low", F64_TYPE),
@@ -83,9 +85,10 @@ public class RustVsJavaWriteBenchmark {
8385
));
8486

8587
private static final DType.Struct JAVA_SCHEMA = new DType.Struct(
86-
List.of("date", "open", "high", "low", "close", "volume"),
88+
List.of("date", "symbol", "open", "high", "low", "close", "volume"),
8789
List.of(
8890
new DType.Primitive(PType.I32, false),
91+
new DType.Utf8(false),
8992
new DType.Primitive(PType.F64, false),
9093
new DType.Primitive(PType.F64, false),
9194
new DType.Primitive(PType.F64, false),
@@ -95,6 +98,21 @@ public class RustVsJavaWriteBenchmark {
9598
false
9699
);
97100

101+
// Real Nasdaq tickers — mix of lengths (1–5 chars) for realistic varbin distribution.
102+
private static final String[] NASDAQ_TICKERS = {
103+
"AAPL", "MSFT", "NVDA", "AMZN", "META", "GOOGL", "TSLA", "AVGO", "COST", "NFLX",
104+
"AMD", "ADBE", "QCOM", "PEP", "CSCO", "TXN", "INTC", "CMCSA", "INTU", "AMGN",
105+
"HON", "AMAT", "MU", "LRCX", "KLAC", "MRVL", "PANW", "SNPS", "CDNS", "REGN"
106+
};
107+
private static final byte[][] TICKER_BYTES;
108+
109+
static {
110+
TICKER_BYTES = new byte[NASDAQ_TICKERS.length][];
111+
for (int i = 0; i < NASDAQ_TICKERS.length; i++) {
112+
TICKER_BYTES[i] = NASDAQ_TICKERS[i].getBytes(StandardCharsets.UTF_8);
113+
}
114+
}
115+
98116
private static final Session SESSION = Session.create();
99117

100118
static {
@@ -103,6 +121,7 @@ public class RustVsJavaWriteBenchmark {
103121

104122
// Pre-generated batch data — filled once in @Setup, reused across invocations.
105123
private int[][] batchDates;
124+
private String[][] batchSymbols;
106125
private double[][] batchOpen;
107126
private double[][] batchHigh;
108127
private double[][] batchLow;
@@ -123,37 +142,39 @@ public void setup() throws IOException {
123142
jniFile = Files.createTempFile("ohlc-jni-write", ".vtx");
124143
javaFile = Files.createTempFile("ohlc-java-write", ".vtx");
125144

126-
batchDates = new int[NUM_BATCHES][BATCH_SIZE];
127-
batchOpen = new double[NUM_BATCHES][BATCH_SIZE];
128-
batchHigh = new double[NUM_BATCHES][BATCH_SIZE];
129-
batchLow = new double[NUM_BATCHES][BATCH_SIZE];
130-
batchClose = new double[NUM_BATCHES][BATCH_SIZE];
131-
batchVolume = new long[NUM_BATCHES][BATCH_SIZE];
145+
batchDates = new int[NUM_BATCHES][BATCH_SIZE];
146+
batchSymbols = new String[NUM_BATCHES][BATCH_SIZE];
147+
batchOpen = new double[NUM_BATCHES][BATCH_SIZE];
148+
batchHigh = new double[NUM_BATCHES][BATCH_SIZE];
149+
batchLow = new double[NUM_BATCHES][BATCH_SIZE];
150+
batchClose = new double[NUM_BATCHES][BATCH_SIZE];
151+
batchVolume = new long[NUM_BATCHES][BATCH_SIZE];
132152

133-
double[] prices = new double[30];
153+
double[] prices = new double[NASDAQ_TICKERS.length];
134154
Arrays.fill(prices, 100.0);
135155
var rng = new Random(42L);
136156
int day = (int) LocalDate.of(2020, 1, 2).toEpochDay();
137157

138158
for (int b = 0; b < NUM_BATCHES; b++) {
139159
for (int i = 0; i < BATCH_SIZE; i++) {
140-
int ticker = i % 30;
160+
int ticker = i % NASDAQ_TICKERS.length;
141161
double px = prices[ticker];
142162
double ret = rng.nextGaussian() * 0.02;
143163
double o = round(px * (1 + ret * 0.3));
144164
double c = round(px * (1 + ret));
145165
double spread = Math.abs(px * rng.nextDouble() * 0.03);
146166
double h = round(Math.max(o, c) + spread);
147167
double l = round(Math.min(o, c) - spread);
148-
batchDates[b][i] = day + (b * BATCH_SIZE + i) / 30;
149-
batchOpen[b][i] = o;
150-
batchHigh[b][i] = h;
151-
batchLow[b][i] = l;
152-
batchClose[b][i] = c;
153-
batchVolume[b][i] = Math.max(100_000L, Math.round(1_000_000 + rng.nextGaussian() * 200_000));
168+
batchDates[b][i] = day + (b * BATCH_SIZE + i) / NASDAQ_TICKERS.length;
169+
batchSymbols[b][i] = NASDAQ_TICKERS[ticker];
170+
batchOpen[b][i] = o;
171+
batchHigh[b][i] = h;
172+
batchLow[b][i] = l;
173+
batchClose[b][i] = c;
174+
batchVolume[b][i] = Math.max(100_000L, Math.round(1_000_000 + rng.nextGaussian() * 200_000));
154175
prices[ticker] = c;
155176
}
156-
day += BATCH_SIZE / 30;
177+
day += BATCH_SIZE / NASDAQ_TICKERS.length;
157178
}
158179

159180
System.out.printf("[RustVsJavaWriteBenchmark] data pre-generated: %d rows in %d batches%n",
@@ -187,6 +208,7 @@ public long javaWrite() throws IOException {
187208
for (int b = 0; b < NUM_BATCHES; b++) {
188209
Map<String, Object> chunk = Map.of(
189210
"date", batchDates[b],
211+
"symbol", batchSymbols[b],
190212
"open", batchOpen[b],
191213
"high", batchHigh[b],
192214
"low", batchLow[b],
@@ -201,15 +223,17 @@ public long javaWrite() throws IOException {
201223

202224
private void flushJni(dev.vortex.api.VortexWriter writer, int b) throws IOException {
203225
try (VectorSchemaRoot root = VectorSchemaRoot.create(JNI_SCHEMA, allocator)) {
204-
DateDayVector dateVec = (DateDayVector) root.getVector("date");
205-
Float8Vector openVec = (Float8Vector) root.getVector("open");
206-
Float8Vector highVec = (Float8Vector) root.getVector("high");
207-
Float8Vector lowVec = (Float8Vector) root.getVector("low");
208-
Float8Vector closeVec = (Float8Vector) root.getVector("close");
209-
BigIntVector volVec = (BigIntVector) root.getVector("volume");
226+
DateDayVector dateVec = (DateDayVector) root.getVector("date");
227+
VarCharVector symbolVec = (VarCharVector) root.getVector("symbol");
228+
Float8Vector openVec = (Float8Vector) root.getVector("open");
229+
Float8Vector highVec = (Float8Vector) root.getVector("high");
230+
Float8Vector lowVec = (Float8Vector) root.getVector("low");
231+
Float8Vector closeVec = (Float8Vector) root.getVector("close");
232+
BigIntVector volVec = (BigIntVector) root.getVector("volume");
210233

211234
int n = batchDates[b].length;
212235
dateVec.allocateNew(n);
236+
symbolVec.allocateNew(n);
213237
openVec.allocateNew(n);
214238
highVec.allocateNew(n);
215239
lowVec.allocateNew(n);
@@ -218,6 +242,7 @@ private void flushJni(dev.vortex.api.VortexWriter writer, int b) throws IOExcept
218242

219243
for (int i = 0; i < n; i++) {
220244
dateVec.setSafe(i, batchDates[b][i]);
245+
symbolVec.setSafe(i, TICKER_BYTES[i % NASDAQ_TICKERS.length]);
221246
openVec.setSafe(i, batchOpen[b][i]);
222247
highVec.setSafe(i, batchHigh[b][i]);
223248
lowVec.setSafe(i, batchLow[b][i]);

0 commit comments

Comments
 (0)