Skip to content

Commit 8a8ddce

Browse files
dfa1claude
dfa1
and
claude
committed
perf: add TaxiLayoutInspector to compare JNI vs Java encoding
Shows Rust writer produces 42.8 MB vs Java's 76.0 MB on NYC taxi data. Key gaps: datetimeparts for timestamps, FSST for strings, sparse/runend for near-constant columns, global dict vs per-chunk dict. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent a283857 commit 8a8ddce

1 file changed

Lines changed: 259 additions & 0 deletions

File tree

‎performance/src/main/java/io/github/dfa1/vortex/performance/TaxiLayoutInspector.java‎

Lines changed: 259 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,259 @@
1+
package io.github.dfa1.vortex.performance;
2+
3+
import dev.hardwood.InputFile;
4+
import dev.hardwood.metadata.RepetitionType;
5+
import dev.hardwood.reader.ParquetFileReader;
6+
import dev.hardwood.reader.RowReader;
7+
import dev.hardwood.schema.ColumnSchema;
8+
import dev.vortex.api.Session;
9+
import dev.vortex.arrow.ArrowAllocation;
10+
import dev.vortex.jni.NativeLoader;
11+
import io.github.dfa1.vortex.encoding.EncodingRegistry;
12+
import io.github.dfa1.vortex.io.VortexInspector;
13+
import io.github.dfa1.vortex.io.VortexReader;
14+
import io.github.dfa1.vortex.parquet.ParquetImporter;
15+
import org.apache.arrow.c.ArrowArray;
16+
import org.apache.arrow.c.ArrowSchema;
17+
import org.apache.arrow.c.Data;
18+
import org.apache.arrow.memory.BufferAllocator;
19+
import org.apache.arrow.vector.BigIntVector;
20+
import org.apache.arrow.vector.Float8Vector;
21+
import org.apache.arrow.vector.IntVector;
22+
import org.apache.arrow.vector.TimeStampMicroVector;
23+
import org.apache.arrow.vector.VarCharVector;
24+
import org.apache.arrow.vector.VectorSchemaRoot;
25+
import org.apache.arrow.vector.types.FloatingPointPrecision;
26+
import org.apache.arrow.vector.types.TimeUnit;
27+
import org.apache.arrow.vector.types.pojo.ArrowType;
28+
import org.apache.arrow.vector.types.pojo.Field;
29+
import org.apache.arrow.vector.types.pojo.Schema;
30+
31+
import java.io.IOException;
32+
import java.io.InputStream;
33+
import java.net.URI;
34+
import java.net.http.HttpClient;
35+
import java.net.http.HttpRequest;
36+
import java.net.http.HttpResponse;
37+
import java.nio.charset.StandardCharsets;
38+
import java.nio.file.Files;
39+
import java.nio.file.Path;
40+
import java.util.HashMap;
41+
import java.util.List;
42+
43+
/// Compare JNI (Rust) vs Java Vortex encoding for the NYC Yellow Taxi 2024-01 dataset.
44+
///
45+
/// Writes the same Parquet file using both writers, then prints {@code inspect} output
46+
/// so encoding choices can be compared side-by-side.
47+
///
48+
/// Run: java -cp performance/target/benchmarks.jar \
49+
/// --enable-native-access=ALL-UNNAMED \
50+
/// --add-opens java.base/java.nio=ALL-UNNAMED \
51+
/// io.github.dfa1.vortex.performance.TaxiLayoutInspector
52+
public final class TaxiLayoutInspector {
53+
54+
private static final String PARQUET_URL =
55+
"https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2024-01.parquet";
56+
private static final Path CACHE_PATH =
57+
Path.of(System.getProperty("java.io.tmpdir"), "yellow_tripdata_2024-01.parquet");
58+
private static final int BATCH_SIZE = 65_536;
59+
private static final Session SESSION = Session.create();
60+
private static final ArrowType F64 =
61+
new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE);
62+
private static final ArrowType I32 = new ArrowType.Int(32, true);
63+
private static final ArrowType I64 = new ArrowType.Int(64, true);
64+
private static final ArrowType TS_MICROS =
65+
new ArrowType.Timestamp(TimeUnit.MICROSECOND, null);
66+
67+
static {
68+
NativeLoader.loadJni();
69+
}
70+
71+
// Arrow schema matching all 19 nullable taxi columns
72+
private static final Schema ARROW_SCHEMA = new Schema(List.of(
73+
Field.nullable("VendorID", I32),
74+
Field.nullable("tpep_pickup_datetime", TS_MICROS),
75+
Field.nullable("tpep_dropoff_datetime", TS_MICROS),
76+
Field.nullable("passenger_count", I64),
77+
Field.nullable("trip_distance", F64),
78+
Field.nullable("RatecodeID", I64),
79+
Field.nullable("store_and_fwd_flag", ArrowType.Utf8.INSTANCE),
80+
Field.nullable("PULocationID", I32),
81+
Field.nullable("DOLocationID", I32),
82+
Field.nullable("payment_type", I64),
83+
Field.nullable("fare_amount", F64),
84+
Field.nullable("extra", F64),
85+
Field.nullable("mta_tax", F64),
86+
Field.nullable("tip_amount", F64),
87+
Field.nullable("tolls_amount", F64),
88+
Field.nullable("improvement_surcharge", F64),
89+
Field.nullable("total_amount", F64),
90+
Field.nullable("congestion_surcharge", F64),
91+
Field.nullable("Airport_fee", F64)
92+
));
93+
94+
private TaxiLayoutInspector() {
95+
}
96+
97+
public static void main(String[] args) throws Exception {
98+
Path parquetFile = ensureCached();
99+
System.out.printf("Parquet: %6.1f MB%n", mb(parquetFile));
100+
101+
Path jniVortex = Files.createTempFile("taxi-jni", ".vortex");
102+
Path javaVortex = Files.createTempFile("taxi-java", ".vortex");
103+
104+
try {
105+
System.out.print("Writing JNI Vortex (Rust encoder)...");
106+
writeJni(parquetFile, jniVortex);
107+
System.out.printf(" %6.1f MB%n", mb(jniVortex));
108+
109+
System.out.print("Writing Java Vortex (cascading depth 3)...");
110+
ParquetImporter.importParquet(parquetFile, javaVortex);
111+
System.out.printf(" %6.1f MB%n", mb(javaVortex));
112+
113+
System.out.println("\n══════════════════════════════════════════════════════");
114+
System.out.println(" JNI / Rust writer");
115+
System.out.println("══════════════════════════════════════════════════════");
116+
inspect(jniVortex);
117+
118+
System.out.println("\n══════════════════════════════════════════════════════");
119+
System.out.println(" Java writer");
120+
System.out.println("══════════════════════════════════════════════════════");
121+
inspect(javaVortex);
122+
} finally {
123+
Files.deleteIfExists(jniVortex);
124+
Files.deleteIfExists(javaVortex);
125+
}
126+
}
127+
128+
private static void inspect(Path path) throws IOException {
129+
try (VortexReader r = VortexReader.open(path, EncodingRegistry.loadAll())) {
130+
System.out.println(VortexInspector.inspect(r));
131+
}
132+
}
133+
134+
private static void writeJni(Path parquetFile, Path vortexFile) throws Exception {
135+
String uri = vortexFile.toAbsolutePath().toUri().toString();
136+
BufferAllocator allocator = ArrowAllocation.rootAllocator();
137+
138+
try (dev.vortex.api.VortexWriter writer = dev.vortex.api.VortexWriter.create(
139+
SESSION, uri, ARROW_SCHEMA, new HashMap<>(), allocator);
140+
ParquetFileReader parquet = ParquetFileReader.open(InputFile.of(parquetFile));
141+
RowReader rows = parquet.buildRowReader().build()) {
142+
143+
List<ColumnSchema> cols = parquet.getFileSchema().getColumns();
144+
boolean[] optional = new boolean[cols.size()];
145+
for (int i = 0; i < cols.size(); i++) {
146+
optional[i] = cols.get(i).repetitionType() == RepetitionType.OPTIONAL;
147+
}
148+
149+
// column name arrays for fast lookup
150+
String[] names = cols.stream().map(ColumnSchema::name).toArray(String[]::new);
151+
152+
try (VectorSchemaRoot root = VectorSchemaRoot.create(ARROW_SCHEMA, allocator)) {
153+
int pos = 0;
154+
allocateVectors(root, BATCH_SIZE);
155+
156+
while (rows.hasNext()) {
157+
rows.next();
158+
fillRow(rows, root, names, optional, pos);
159+
pos++;
160+
if (pos == BATCH_SIZE) {
161+
flushBatch(writer, allocator, root, pos);
162+
allocateVectors(root, BATCH_SIZE);
163+
pos = 0;
164+
}
165+
}
166+
if (pos > 0) {
167+
flushBatch(writer, allocator, root, pos);
168+
}
169+
}
170+
}
171+
}
172+
173+
private static void allocateVectors(VectorSchemaRoot root, int capacity) {
174+
for (var vec : root.getFieldVectors()) {
175+
vec.allocateNew();
176+
}
177+
}
178+
179+
private static void fillRow(RowReader rows, VectorSchemaRoot root,
180+
String[] names, boolean[] optional, int pos) {
181+
for (int c = 0; c < names.length; c++) {
182+
String name = names[c];
183+
boolean isNull = optional[c] && rows.isNull(name);
184+
switch (root.getVector(name)) {
185+
case IntVector v -> {
186+
if (isNull) {
187+
v.setNull(pos);
188+
} else {
189+
v.setSafe(pos, rows.getInt(name));
190+
}
191+
}
192+
case BigIntVector v -> {
193+
if (isNull) {
194+
v.setNull(pos);
195+
} else {
196+
v.setSafe(pos, rows.getLong(name));
197+
}
198+
}
199+
case TimeStampMicroVector v -> {
200+
if (isNull) {
201+
v.setNull(pos);
202+
} else {
203+
v.setSafe(pos, rows.getLong(name));
204+
}
205+
}
206+
case Float8Vector v -> {
207+
if (isNull) {
208+
v.setNull(pos);
209+
} else {
210+
v.setSafe(pos, rows.getDouble(name));
211+
}
212+
}
213+
case VarCharVector v -> {
214+
if (isNull) {
215+
v.setNull(pos);
216+
} else {
217+
byte[] bytes = rows.getString(name).getBytes(StandardCharsets.UTF_8);
218+
v.setSafe(pos, bytes);
219+
}
220+
}
221+
default -> throw new UnsupportedOperationException(
222+
"unexpected vector type: " + root.getVector(name).getClass().getSimpleName());
223+
}
224+
}
225+
}
226+
227+
private static void flushBatch(dev.vortex.api.VortexWriter writer,
228+
BufferAllocator allocator,
229+
VectorSchemaRoot root, int rowCount) throws IOException {
230+
root.setRowCount(rowCount);
231+
try (ArrowArray arr = ArrowArray.allocateNew(allocator);
232+
ArrowSchema schema = ArrowSchema.allocateNew(allocator)) {
233+
Data.exportVectorSchemaRoot(allocator, root, null, arr, schema);
234+
writer.writeBatch(arr.memoryAddress(), schema.memoryAddress());
235+
}
236+
}
237+
238+
private static Path ensureCached() throws Exception {
239+
if (Files.exists(CACHE_PATH)) {
240+
System.out.printf("cache hit: %s%n", CACHE_PATH);
241+
return CACHE_PATH;
242+
}
243+
System.out.printf("downloading %s...%n", PARQUET_URL);
244+
HttpClient client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build();
245+
HttpRequest req = HttpRequest.newBuilder().uri(URI.create(PARQUET_URL)).build();
246+
HttpResponse<InputStream> resp = client.send(req, HttpResponse.BodyHandlers.ofInputStream());
247+
if (resp.statusCode() != 200) {
248+
throw new IOException("HTTP " + resp.statusCode());
249+
}
250+
try (InputStream in = resp.body()) {
251+
Files.copy(in, CACHE_PATH);
252+
}
253+
return CACHE_PATH;
254+
}
255+
256+
private static double mb(Path path) throws IOException {
257+
return Files.size(path) / 1_048_576.0;
258+
}
259+
}

0 commit comments

Comments
 (0)