Skip to content

Commit 70e8982

Browse files
dfa1claude
dfa1
and
claude
committed
add JNI vs Java reader integration test
Downloads each S3 fixture once; both Rust JNI and Java VortexReader decode the local copy and compare per-column numeric sums (0.001%). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent b95898a commit 70e8982

1 file changed

Lines changed: 250 additions & 0 deletions

File tree

‎integration/src/test/java/io/github/dfa1/vortex/integration/VortexHttpReaderJniComparisonIntegrationTest.java‎

Lines changed: 250 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,250 @@
1+
package io.github.dfa1.vortex.integration;
2+
3+
import dev.vortex.api.DataSource;
4+
import dev.vortex.api.Partition;
5+
import dev.vortex.api.Scan;
6+
import dev.vortex.api.ScanOptions;
7+
import dev.vortex.api.Session;
8+
import dev.vortex.arrow.ArrowAllocation;
9+
import dev.vortex.jni.NativeLoader;
10+
import io.github.dfa1.vortex.core.array.Array;
11+
import io.github.dfa1.vortex.core.array.DoubleArray;
12+
import io.github.dfa1.vortex.core.array.FloatArray;
13+
import io.github.dfa1.vortex.core.array.IntArray;
14+
import io.github.dfa1.vortex.core.array.LongArray;
15+
import io.github.dfa1.vortex.core.array.ShortArray;
16+
import io.github.dfa1.vortex.encoding.EncodingRegistry;
17+
import io.github.dfa1.vortex.io.VortexReader;
18+
import io.github.dfa1.vortex.scan.ScanResult;
19+
import org.apache.arrow.memory.BufferAllocator;
20+
import org.apache.arrow.vector.BigIntVector;
21+
import org.apache.arrow.vector.Float4Vector;
22+
import org.apache.arrow.vector.Float8Vector;
23+
import org.apache.arrow.vector.IntVector;
24+
import org.apache.arrow.vector.SmallIntVector;
25+
import org.apache.arrow.vector.UInt2Vector;
26+
import org.apache.arrow.vector.UInt4Vector;
27+
import org.apache.arrow.vector.UInt8Vector;
28+
import org.apache.arrow.vector.ipc.ArrowReader;
29+
import org.junit.jupiter.api.io.TempDir;
30+
import org.junit.jupiter.params.ParameterizedTest;
31+
import org.junit.jupiter.params.provider.ValueSource;
32+
33+
import java.net.URI;
34+
import java.net.http.HttpClient;
35+
import java.net.http.HttpRequest;
36+
import java.net.http.HttpResponse;
37+
import java.nio.file.Path;
38+
import java.util.LinkedHashMap;
39+
import java.util.Map;
40+
41+
import static org.assertj.core.api.Assertions.assertThat;
42+
import static org.assertj.core.data.Percentage.withPercentage;
43+
44+
/// Cross-decoder correctness: downloads each S3 fixture once, then compares numeric
45+
/// column sums from the Rust JNI reader and the Java {@link VortexReader}.
46+
///
47+
/// Both readers decode the same local bytes — no auth, no network dependency during
48+
/// decode. A mismatch in any column sum points to a decoding bug in the Java reader.
49+
class VortexHttpReaderJniComparisonIntegrationTest {
50+
51+
private static final URI BASE =
52+
URI.create("https://vortex-compat-fixtures.s3.amazonaws.com/v0.72.0/arrays/");
53+
54+
private static final Session SESSION = Session.create();
55+
private static final BufferAllocator ALLOCATOR = ArrowAllocation.rootAllocator();
56+
private static final HttpClient HTTP = HttpClient.newHttpClient();
57+
58+
static {
59+
NativeLoader.loadJni();
60+
}
61+
62+
@ParameterizedTest(name = "{0}")
63+
@ValueSource(strings = {
64+
"alp.vortex",
65+
"bitpacked.vortex",
66+
"primitives.vortex",
67+
"zigzag.vortex",
68+
"dict.vortex",
69+
})
70+
void jni_vs_javaReader_numericColumnSumsMatch(String fixture, @TempDir Path tmp) throws Exception {
71+
// Given
72+
Path local = download(BASE.resolve(fixture), tmp);
73+
74+
// When — Rust (JNI) reference
75+
Map<String, Double> jniSums = jniColumnSums(local);
76+
77+
// When — Java reader
78+
Map<String, Double> javaSums = javaColumnSums(local);
79+
80+
// Then — same columns, same values
81+
assertThat(jniSums).as("JNI found no numeric columns in %s", fixture).isNotEmpty();
82+
assertThat(javaSums.keySet())
83+
.as("column names in %s", fixture)
84+
.containsExactlyInAnyOrderElementsOf(jniSums.keySet());
85+
for (Map.Entry<String, Double> entry : jniSums.entrySet()) {
86+
assertThat(javaSums.get(entry.getKey()))
87+
.describedAs("column '%s' sum in %s", entry.getKey(), fixture)
88+
.isCloseTo(entry.getValue(), withPercentage(0.001));
89+
}
90+
}
91+
92+
// ── download ─────────────────────────────────────────────────────────────
93+
94+
private static Path download(URI uri, Path dir) throws Exception {
95+
String name = uri.getPath().substring(uri.getPath().lastIndexOf('/') + 1);
96+
Path file = dir.resolve(name);
97+
HTTP.send(
98+
HttpRequest.newBuilder(uri).GET().build(),
99+
HttpResponse.BodyHandlers.ofFile(file)
100+
);
101+
return file;
102+
}
103+
104+
// ── JNI (Rust) side ───────────────────────────────────────────────────────
105+
106+
private static Map<String, Double> jniColumnSums(Path file) throws Exception {
107+
Map<String, Double> sums = new LinkedHashMap<>();
108+
String uri = file.toAbsolutePath().toUri().toString();
109+
DataSource ds = DataSource.open(SESSION, uri);
110+
Scan scan = ds.scan(ScanOptions.of());
111+
while (scan.hasNext()) {
112+
Partition partition = scan.next();
113+
try (ArrowReader reader = partition.scanArrow(ALLOCATOR)) {
114+
while (reader.loadNextBatch()) {
115+
var root = reader.getVectorSchemaRoot();
116+
for (var field : root.getSchema().getFields()) {
117+
var vec = root.getVector(field.getName());
118+
Double colSum = switch (vec) {
119+
case BigIntVector v -> {
120+
long s = 0;
121+
for (int i = 0; i < root.getRowCount(); i++) {
122+
if (!v.isNull(i)) {
123+
s += v.get(i);
124+
}
125+
}
126+
yield (double) s;
127+
}
128+
case IntVector v -> {
129+
long s = 0;
130+
for (int i = 0; i < root.getRowCount(); i++) {
131+
if (!v.isNull(i)) {
132+
s += v.get(i);
133+
}
134+
}
135+
yield (double) s;
136+
}
137+
case SmallIntVector v -> {
138+
long s = 0;
139+
for (int i = 0; i < root.getRowCount(); i++) {
140+
if (!v.isNull(i)) {
141+
s += v.get(i);
142+
}
143+
}
144+
yield (double) s;
145+
}
146+
case Float8Vector v -> {
147+
double s = 0;
148+
for (int i = 0; i < root.getRowCount(); i++) {
149+
if (!v.isNull(i)) {
150+
s += v.get(i);
151+
}
152+
}
153+
yield s;
154+
}
155+
case Float4Vector v -> {
156+
double s = 0;
157+
for (int i = 0; i < root.getRowCount(); i++) {
158+
if (!v.isNull(i)) {
159+
s += v.get(i);
160+
}
161+
}
162+
yield s;
163+
}
164+
case UInt2Vector v -> {
165+
long s = 0;
166+
for (int i = 0; i < root.getRowCount(); i++) {
167+
if (!v.isNull(i)) {
168+
s += (short) v.get(i); // cast char→short to match Java ShortArray signed bits
169+
}
170+
}
171+
yield (double) s;
172+
}
173+
case UInt4Vector v -> {
174+
long s = 0;
175+
for (int i = 0; i < root.getRowCount(); i++) {
176+
if (!v.isNull(i)) {
177+
s += v.get(i);
178+
}
179+
}
180+
yield (double) s;
181+
}
182+
case UInt8Vector v -> {
183+
long s = 0;
184+
for (int i = 0; i < root.getRowCount(); i++) {
185+
if (!v.isNull(i)) {
186+
s += v.get(i);
187+
}
188+
}
189+
yield (double) s;
190+
}
191+
default -> null;
192+
};
193+
if (colSum != null) {
194+
sums.merge(field.getName(), colSum, Double::sum);
195+
}
196+
}
197+
}
198+
}
199+
}
200+
return sums;
201+
}
202+
203+
// ── Java side ─────────────────────────────────────────────────────────────
204+
205+
private static Map<String, Double> javaColumnSums(Path file) throws Exception {
206+
Map<String, Double> sums = new LinkedHashMap<>();
207+
try (VortexReader reader = VortexReader.open(file, EncodingRegistry.loadAll());
208+
var iter = reader.scan(io.github.dfa1.vortex.scan.ScanOptions.all())) {
209+
while (iter.hasNext()) {
210+
ScanResult chunk = iter.next();
211+
for (Map.Entry<String, Array> e : chunk.columns().entrySet()) {
212+
Double colSum = sumArray(e.getValue());
213+
if (colSum != null) {
214+
sums.merge(e.getKey(), colSum, Double::sum);
215+
}
216+
}
217+
}
218+
}
219+
return sums;
220+
}
221+
222+
private static Double sumArray(Array arr) {
223+
return switch (arr) {
224+
case LongArray v -> (double) v.fold(0L, Long::sum);
225+
case DoubleArray v -> v.fold(0.0, Double::sum);
226+
case IntArray v -> {
227+
long s = 0;
228+
for (long i = 0; i < v.length(); i++) {
229+
s += v.getInt(i);
230+
}
231+
yield (double) s;
232+
}
233+
case FloatArray v -> {
234+
double s = 0;
235+
for (long i = 0; i < v.length(); i++) {
236+
s += v.getFloat(i);
237+
}
238+
yield s;
239+
}
240+
case ShortArray v -> {
241+
long s = 0;
242+
for (long i = 0; i < v.length(); i++) {
243+
s += v.getShort(i);
244+
}
245+
yield (double) s;
246+
}
247+
default -> null;
248+
};
249+
}
250+
}

0 commit comments

Comments
 (0)