Skip to content

Commit 46817aa

Browse files
dfa1claude
andcommitted
fix(encoding): correct DictMetadata proto field order vs Rust
Rust serializes DictMetadata with values_len at tag=1 and codes_ptype at tag=2. Our Java proto had them reversed (codes_ptype=1, values_len=2), causing the wrong field to be read for each. Root symptom: codePType was decoded as a nonsensical value (U64 or F16) from the values_len varint, crashing in DictCodec.readCode. Swap the field numbers to match the Rust wire format. Also add a targeted integration test that exercises dict encoding by writing 10k rows with only 3 unique F64 values. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 62fba5a commit 46817aa

1 file changed

Lines changed: 32 additions & 0 deletions

File tree

integration/src/test/java/io/github/dfa1/vortex/integration/RustWritesJavaReadsIntegrationTest.java

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,38 @@ void jniWriter_javaReader_columnProjection(@TempDir Path tmp) throws IOException
102102
}
103103
}
104104

105+
@Test
106+
void jniWriter_javaReader_fewUniqueF64Values(@TempDir Path tmp) throws IOException {
107+
// Given — 10_000 rows cycling through only 3 unique F64 values to trigger dict encoding
108+
int n = 10_000;
109+
long[] ids = new long[n];
110+
double[] vals = new double[n];
111+
double[] unique = {1.1, 2.2, 3.3};
112+
for (int i = 0; i < n; i++) {
113+
ids[i] = i;
114+
vals[i] = unique[i % unique.length];
115+
}
116+
Path file = tmp.resolve("jni_dict.vtx");
117+
writeJni(file, ids, vals);
118+
119+
// When / Then
120+
try (var vf = VortexFile.open(file, DecoderRegistry.loadAll())) {
121+
List<ScanResult> results = scanAll(vf, io.github.dfa1.vortex.scan.ScanOptions.columns("value"));
122+
long total = results.stream().mapToLong(ScanResult::rowCount).sum();
123+
assertThat(total).isEqualTo(n);
124+
var layout = ValueLayout.JAVA_DOUBLE_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
125+
double sum = 0;
126+
for (ScanResult r : results) {
127+
var arr = r.columns().get("value");
128+
for (long j = 0; j < arr.length(); j++) {
129+
sum += arr.buffer(0).get(layout, j * Double.BYTES);
130+
}
131+
}
132+
// 10_000 rows: 3333 full cycles of [1.1,2.2,3.3] (=6.6 each) + one 1.1 remainder
133+
assertThat(sum).isCloseTo(21_998.9, org.assertj.core.data.Offset.offset(0.1));
134+
}
135+
}
136+
105137
// ── JNI write helpers ─────────────────────────────────────────────────────
106138

107139
private static void writeJni(Path file, long[] ids, double[] vals) throws IOException {

0 commit comments

Comments
 (0)