Skip to content

Commit 8d60d95

Browse files
dfa1claude
andcommitted
test(integration): add NaN/Inf cross-compat tests for F64, F32, and F16
F64/F32: NaN and Inf go to ALP patches (raw bits). containsExactly breaks on NaN (IEEE 754: NaN != NaN), so tests use assertBitwiseEqualsF64/F32 via doubleToRawLongBits/floatToRawIntBits. F16: PrimitiveEncoding stores raw short bits directly — NaN and Inf are opaque 16-bit patterns. Test is @disabled because vortex-jni 0.72 cannot export F16 columns via the Arrow C Data Interface. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent a73cf56 commit 8d60d95

1 file changed

Lines changed: 106 additions & 0 deletions

File tree

integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,13 @@
2020
import net.jqwik.api.Provide;
2121
import org.apache.arrow.memory.BufferAllocator;
2222
import org.apache.arrow.vector.BigIntVector;
23+
import org.apache.arrow.vector.Float2Vector;
2324
import org.apache.arrow.vector.Float8Vector;
2425
import org.apache.arrow.vector.IntVector;
2526
import org.apache.arrow.vector.VarCharVector;
2627
import org.apache.arrow.vector.VectorSchemaRoot;
2728
import org.apache.arrow.vector.ipc.ArrowReader;
29+
import org.junit.jupiter.api.Disabled;
2830
import org.junit.jupiter.api.Test;
2931
import org.junit.jupiter.api.io.TempDir;
3032

@@ -77,6 +79,11 @@ class JavaWritesRustReadsIntegrationTest {
7779
List.of(new DType.Primitive(PType.F64, false)),
7880
false);
7981

82+
private static final DType.Struct F16_SCHEMA = new DType.Struct(
83+
List.of("v"),
84+
List.of(new DType.Primitive(PType.F16, false)),
85+
false);
86+
8087
private static final DType.Struct OHLC_SCHEMA = new DType.Struct(
8188
List.of("date", "symbol", "open", "high", "low", "close", "volume"),
8289
List.of(
@@ -166,6 +173,33 @@ private static float[] readFloatColumn(Path file, String column) throws IOExcept
166173
return result;
167174
}
168175

176+
private static short[] readHalfColumn(Path file, String column) throws IOException {
177+
String uri = file.toAbsolutePath().toUri().toString();
178+
ScanOptions opts = ScanOptions.builder()
179+
.projection(Expression.select(new String[]{column}, Expression.root()))
180+
.build();
181+
var halfs = new ArrayList<Short>();
182+
DataSource ds = DataSource.open(SESSION, uri);
183+
Scan scan = ds.scan(opts);
184+
while (scan.hasNext()) {
185+
Partition partition = scan.next();
186+
try (ArrowReader reader = partition.scanArrow(ALLOCATOR)) {
187+
while (reader.loadNextBatch()) {
188+
VectorSchemaRoot root = reader.getVectorSchemaRoot();
189+
Float2Vector vec = (Float2Vector) root.getVector(column);
190+
for (int i = 0; i < root.getRowCount(); i++) {
191+
halfs.add(vec.get(i));
192+
}
193+
}
194+
}
195+
}
196+
short[] result = new short[halfs.size()];
197+
for (int i = 0; i < result.length; i++) {
198+
result[i] = halfs.get(i);
199+
}
200+
return result;
201+
}
202+
169203
private static int[] readIntColumn(Path file, String column) throws IOException {
170204
String uri = file.toAbsolutePath().toUri().toString();
171205
ScanOptions opts = ScanOptions.builder()
@@ -555,6 +589,78 @@ Arbitrary<float[]> f32Arrays() {
555589
.ofMinSize(0).ofMaxSize(5_000);
556590
}
557591

592+
/// F64: NaN and Inf go to ALP patches (raw bits). containsExactly fails for NaN — use bitwise comparison.
593+
@Test
594+
void javaWriter_jniReader_f64_nanAndInf(@TempDir Path tmp) throws IOException {
595+
// Given
596+
Path file = tmp.resolve("f64_nan_inf.vtx");
597+
double[] data = {Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY, 0.0, 1.5, -1.5};
598+
try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
599+
var sut = VortexWriter.create(ch, F64_SCHEMA, WriteOptions.defaults())) {
600+
// When
601+
sut.writeChunk(Map.of("v", data));
602+
}
603+
604+
// Then
605+
double[] decoded = readDoubleColumn(file, "v");
606+
assertBitwiseEqualsF64(decoded, data);
607+
}
608+
609+
/// F32: same as F64 — NaN and Inf go to ALP patches, require bitwise comparison.
610+
@Test
611+
void javaWriter_jniReader_f32_nanAndInf(@TempDir Path tmp) throws IOException {
612+
// Given
613+
Path file = tmp.resolve("f32_nan_inf.vtx");
614+
float[] data = {Float.NaN, Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY, 0.0f, 1.5f, -1.5f};
615+
try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
616+
var sut = VortexWriter.create(ch, F32_SCHEMA, WriteOptions.defaults())) {
617+
// When
618+
sut.writeChunk(Map.of("v", data));
619+
}
620+
621+
// Then
622+
float[] decoded = readFloatColumn(file, "v");
623+
assertBitwiseEqualsF32(decoded, data);
624+
}
625+
626+
/// F16: PrimitiveEncoding stores raw short bits — NaN and Inf are just bit patterns.
627+
/// Disabled: vortex-jni 0.72 does not export F16 via Arrow C Data Interface.
628+
@Disabled
629+
@Test
630+
void javaWriter_jniReader_f16_nanAndInf(@TempDir Path tmp) throws IOException {
631+
// Given
632+
Path file = tmp.resolve("f16_nan_inf.vtx");
633+
// Half-precision: +Inf=0x7C00, -Inf=0xFC00, quiet NaN=0x7E00, 1.0=0x3C00, 0.0=0x0000
634+
short[] data = {(short) 0x7E00, (short) 0x7C00, (short) 0xFC00, (short) 0x3C00, (short) 0x0000};
635+
try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
636+
var sut = VortexWriter.create(ch, F16_SCHEMA, WriteOptions.defaults())) {
637+
// When
638+
sut.writeChunk(Map.of("v", data));
639+
}
640+
641+
// Then
642+
short[] decoded = readHalfColumn(file, "v");
643+
assertThat(decoded).containsExactly(data);
644+
}
645+
646+
private static void assertBitwiseEqualsF64(double[] actual, double[] expected) {
647+
assertThat(actual).hasSize(expected.length);
648+
for (int i = 0; i < expected.length; i++) {
649+
assertThat(Double.doubleToRawLongBits(actual[i]))
650+
.as("element %d", i)
651+
.isEqualTo(Double.doubleToRawLongBits(expected[i]));
652+
}
653+
}
654+
655+
private static void assertBitwiseEqualsF32(float[] actual, float[] expected) {
656+
assertThat(actual).hasSize(expected.length);
657+
for (int i = 0; i < expected.length; i++) {
658+
assertThat(Float.floatToRawIntBits(actual[i]))
659+
.as("element %d", i)
660+
.isEqualTo(Float.floatToRawIntBits(expected[i]));
661+
}
662+
}
663+
558664
private static void deleteDir(Path dir) throws IOException {
559665
try (var walk = Files.walk(dir)) {
560666
walk.sorted(Comparator.reverseOrder()).forEach(p -> p.toFile().delete());

0 commit comments

Comments
 (0)