|
20 | 20 | import net.jqwik.api.Provide; |
21 | 21 | import org.apache.arrow.memory.BufferAllocator; |
22 | 22 | import org.apache.arrow.vector.BigIntVector; |
| 23 | +import org.apache.arrow.vector.Float2Vector; |
23 | 24 | import org.apache.arrow.vector.Float8Vector; |
24 | 25 | import org.apache.arrow.vector.IntVector; |
25 | 26 | import org.apache.arrow.vector.VarCharVector; |
26 | 27 | import org.apache.arrow.vector.VectorSchemaRoot; |
27 | 28 | import org.apache.arrow.vector.ipc.ArrowReader; |
| 29 | +import org.junit.jupiter.api.Disabled; |
28 | 30 | import org.junit.jupiter.api.Test; |
29 | 31 | import org.junit.jupiter.api.io.TempDir; |
30 | 32 |
|
@@ -77,6 +79,11 @@ class JavaWritesRustReadsIntegrationTest { |
77 | 79 | List.of(new DType.Primitive(PType.F64, false)), |
78 | 80 | false); |
79 | 81 |
|
| 82 | + private static final DType.Struct F16_SCHEMA = new DType.Struct( |
| 83 | + List.of("v"), |
| 84 | + List.of(new DType.Primitive(PType.F16, false)), |
| 85 | + false); |
| 86 | + |
80 | 87 | private static final DType.Struct OHLC_SCHEMA = new DType.Struct( |
81 | 88 | List.of("date", "symbol", "open", "high", "low", "close", "volume"), |
82 | 89 | List.of( |
@@ -166,6 +173,33 @@ private static float[] readFloatColumn(Path file, String column) throws IOExcept |
166 | 173 | return result; |
167 | 174 | } |
168 | 175 |
|
| 176 | + private static short[] readHalfColumn(Path file, String column) throws IOException { |
| 177 | + String uri = file.toAbsolutePath().toUri().toString(); |
| 178 | + ScanOptions opts = ScanOptions.builder() |
| 179 | + .projection(Expression.select(new String[]{column}, Expression.root())) |
| 180 | + .build(); |
| 181 | + var halfs = new ArrayList<Short>(); |
| 182 | + DataSource ds = DataSource.open(SESSION, uri); |
| 183 | + Scan scan = ds.scan(opts); |
| 184 | + while (scan.hasNext()) { |
| 185 | + Partition partition = scan.next(); |
| 186 | + try (ArrowReader reader = partition.scanArrow(ALLOCATOR)) { |
| 187 | + while (reader.loadNextBatch()) { |
| 188 | + VectorSchemaRoot root = reader.getVectorSchemaRoot(); |
| 189 | + Float2Vector vec = (Float2Vector) root.getVector(column); |
| 190 | + for (int i = 0; i < root.getRowCount(); i++) { |
| 191 | + halfs.add(vec.get(i)); |
| 192 | + } |
| 193 | + } |
| 194 | + } |
| 195 | + } |
| 196 | + short[] result = new short[halfs.size()]; |
| 197 | + for (int i = 0; i < result.length; i++) { |
| 198 | + result[i] = halfs.get(i); |
| 199 | + } |
| 200 | + return result; |
| 201 | + } |
| 202 | + |
169 | 203 | private static int[] readIntColumn(Path file, String column) throws IOException { |
170 | 204 | String uri = file.toAbsolutePath().toUri().toString(); |
171 | 205 | ScanOptions opts = ScanOptions.builder() |
@@ -555,6 +589,78 @@ Arbitrary<float[]> f32Arrays() { |
555 | 589 | .ofMinSize(0).ofMaxSize(5_000); |
556 | 590 | } |
557 | 591 |
|
| 592 | + /// F64: NaN and Inf go to ALP patches (raw bits). containsExactly fails for NaN — use bitwise comparison. |
| 593 | + @Test |
| 594 | + void javaWriter_jniReader_f64_nanAndInf(@TempDir Path tmp) throws IOException { |
| 595 | + // Given |
| 596 | + Path file = tmp.resolve("f64_nan_inf.vtx"); |
| 597 | + double[] data = {Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY, 0.0, 1.5, -1.5}; |
| 598 | + try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); |
| 599 | + var sut = VortexWriter.create(ch, F64_SCHEMA, WriteOptions.defaults())) { |
| 600 | + // When |
| 601 | + sut.writeChunk(Map.of("v", data)); |
| 602 | + } |
| 603 | + |
| 604 | + // Then |
| 605 | + double[] decoded = readDoubleColumn(file, "v"); |
| 606 | + assertBitwiseEqualsF64(decoded, data); |
| 607 | + } |
| 608 | + |
| 609 | + /// F32: same as F64 — NaN and Inf go to ALP patches, require bitwise comparison. |
| 610 | + @Test |
| 611 | + void javaWriter_jniReader_f32_nanAndInf(@TempDir Path tmp) throws IOException { |
| 612 | + // Given |
| 613 | + Path file = tmp.resolve("f32_nan_inf.vtx"); |
| 614 | + float[] data = {Float.NaN, Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY, 0.0f, 1.5f, -1.5f}; |
| 615 | + try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); |
| 616 | + var sut = VortexWriter.create(ch, F32_SCHEMA, WriteOptions.defaults())) { |
| 617 | + // When |
| 618 | + sut.writeChunk(Map.of("v", data)); |
| 619 | + } |
| 620 | + |
| 621 | + // Then |
| 622 | + float[] decoded = readFloatColumn(file, "v"); |
| 623 | + assertBitwiseEqualsF32(decoded, data); |
| 624 | + } |
| 625 | + |
| 626 | + /// F16: PrimitiveEncoding stores raw short bits — NaN and Inf are just bit patterns. |
| 627 | + /// Disabled: vortex-jni 0.72 does not export F16 via Arrow C Data Interface. |
| 628 | + @Disabled |
| 629 | + @Test |
| 630 | + void javaWriter_jniReader_f16_nanAndInf(@TempDir Path tmp) throws IOException { |
| 631 | + // Given |
| 632 | + Path file = tmp.resolve("f16_nan_inf.vtx"); |
| 633 | + // Half-precision: +Inf=0x7C00, -Inf=0xFC00, quiet NaN=0x7E00, 1.0=0x3C00, 0.0=0x0000 |
| 634 | + short[] data = {(short) 0x7E00, (short) 0x7C00, (short) 0xFC00, (short) 0x3C00, (short) 0x0000}; |
| 635 | + try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); |
| 636 | + var sut = VortexWriter.create(ch, F16_SCHEMA, WriteOptions.defaults())) { |
| 637 | + // When |
| 638 | + sut.writeChunk(Map.of("v", data)); |
| 639 | + } |
| 640 | + |
| 641 | + // Then |
| 642 | + short[] decoded = readHalfColumn(file, "v"); |
| 643 | + assertThat(decoded).containsExactly(data); |
| 644 | + } |
| 645 | + |
| 646 | + private static void assertBitwiseEqualsF64(double[] actual, double[] expected) { |
| 647 | + assertThat(actual).hasSize(expected.length); |
| 648 | + for (int i = 0; i < expected.length; i++) { |
| 649 | + assertThat(Double.doubleToRawLongBits(actual[i])) |
| 650 | + .as("element %d", i) |
| 651 | + .isEqualTo(Double.doubleToRawLongBits(expected[i])); |
| 652 | + } |
| 653 | + } |
| 654 | + |
| 655 | + private static void assertBitwiseEqualsF32(float[] actual, float[] expected) { |
| 656 | + assertThat(actual).hasSize(expected.length); |
| 657 | + for (int i = 0; i < expected.length; i++) { |
| 658 | + assertThat(Float.floatToRawIntBits(actual[i])) |
| 659 | + .as("element %d", i) |
| 660 | + .isEqualTo(Float.floatToRawIntBits(expected[i])); |
| 661 | + } |
| 662 | + } |
| 663 | + |
558 | 664 | private static void deleteDir(Path dir) throws IOException { |
559 | 665 | try (var walk = Files.walk(dir)) { |
560 | 666 | walk.sorted(Comparator.reverseOrder()).forEach(p -> p.toFile().delete()); |
|
0 commit comments