|
| 1 | +// SPDX-License-Identifier: Apache-2.0 |
| 2 | +// SPDX-FileCopyrightText: Copyright the Vortex contributors |
| 3 | + |
| 4 | +package dev.vortex.spark.read; |
| 5 | + |
| 6 | +import static org.junit.jupiter.api.Assertions.assertEquals; |
| 7 | +import static org.junit.jupiter.api.Assertions.assertInstanceOf; |
| 8 | + |
| 9 | +import dev.vortex.spark.VortexFilePartition; |
| 10 | +import java.util.List; |
| 11 | +import java.util.Map; |
| 12 | +import org.apache.spark.sql.connector.catalog.Column; |
| 13 | +import org.apache.spark.sql.connector.expressions.filter.Predicate; |
| 14 | +import org.apache.spark.sql.connector.read.InputPartition; |
| 15 | +import org.junit.jupiter.api.DisplayName; |
| 16 | +import org.junit.jupiter.api.Test; |
| 17 | + |
| 18 | +/** |
| 19 | + * Unit tests for {@link VortexBatchExec} input-partition planning. |
| 20 | + * |
| 21 | + * <p>Characterizes how explicit {@code .vortex} paths are planned: one {@link VortexFilePartition} per file, each |
| 22 | + * carrying the requested read schema and the Hive-style partition values parsed from its own path. All paths used here |
| 23 | + * end in {@code .vortex} so planning never lists directories through native I/O. |
| 24 | + */ |
| 25 | +final class VortexBatchExecTest { |
| 26 | + |
| 27 | + private static final List<Column> COLUMNS = List.of( |
| 28 | + Column.create("id", org.apache.spark.sql.types.DataTypes.IntegerType), |
| 29 | + Column.create("name", org.apache.spark.sql.types.DataTypes.StringType)); |
| 30 | + |
| 31 | + private static VortexBatchExec execFor(List<String> paths) { |
| 32 | + return new VortexBatchExec(paths, COLUMNS, Map.of(), new Predicate[0]); |
| 33 | + } |
| 34 | + |
| 35 | + @Test |
| 36 | + @DisplayName("Plans one input partition per .vortex file") |
| 37 | + void plansOnePartitionPerFile() { |
| 38 | + VortexBatchExec exec = execFor(List.of("/data/a.vortex", "/data/b.vortex", "/data/c.vortex")); |
| 39 | + |
| 40 | + InputPartition[] partitions = exec.planInputPartitions(); |
| 41 | + |
| 42 | + assertEquals(3, partitions.length); |
| 43 | + for (InputPartition partition : partitions) { |
| 44 | + assertInstanceOf(VortexFilePartition.class, partition); |
| 45 | + assertEquals(1, ((VortexFilePartition) partition).paths().size()); |
| 46 | + } |
| 47 | + } |
| 48 | + |
| 49 | + @Test |
| 50 | + @DisplayName("Each partition carries exactly its own file path, in input order") |
| 51 | + void partitionsKeepInputOrder() { |
| 52 | + VortexBatchExec exec = execFor(List.of("/data/first.vortex", "/data/second.vortex")); |
| 53 | + |
| 54 | + InputPartition[] partitions = exec.planInputPartitions(); |
| 55 | + |
| 56 | + assertEquals(List.of("/data/first.vortex"), ((VortexFilePartition) partitions[0]).paths()); |
| 57 | + assertEquals(List.of("/data/second.vortex"), ((VortexFilePartition) partitions[1]).paths()); |
| 58 | + } |
| 59 | + |
| 60 | + @Test |
| 61 | + @DisplayName("Partitions carry the requested read schema") |
| 62 | + void partitionsCarryReadSchema() { |
| 63 | + VortexBatchExec exec = execFor(List.of("/data/a.vortex")); |
| 64 | + |
| 65 | + VortexFilePartition partition = (VortexFilePartition) exec.planInputPartitions()[0]; |
| 66 | + |
| 67 | + assertEquals(2, partition.readSchema().size()); |
| 68 | + assertEquals("id", partition.readSchema().fields()[0].name()); |
| 69 | + assertEquals("name", partition.readSchema().fields()[1].name()); |
| 70 | + } |
| 71 | + |
| 72 | + @Test |
| 73 | + @DisplayName("Hive-style partition values are parsed from each file's own path") |
| 74 | + void partitionValuesParsedPerFile() { |
| 75 | + VortexBatchExec exec = execFor( |
| 76 | + List.of("/tbl/year=2024/month=01/a.vortex", "/tbl/year=2025/month=02/b.vortex", "/tbl/plain.vortex")); |
| 77 | + |
| 78 | + InputPartition[] partitions = exec.planInputPartitions(); |
| 79 | + |
| 80 | + assertEquals(Map.of("year", "2024", "month", "01"), ((VortexFilePartition) partitions[0]).partitionValues()); |
| 81 | + assertEquals(Map.of("year", "2025", "month", "02"), ((VortexFilePartition) partitions[1]).partitionValues()); |
| 82 | + assertEquals(Map.of(), ((VortexFilePartition) partitions[2]).partitionValues()); |
| 83 | + } |
| 84 | + |
| 85 | + @Test |
| 86 | + @DisplayName("Format options are propagated to every partition") |
| 87 | + void formatOptionsPropagated() { |
| 88 | + Map<String, String> options = Map.of("vortex.workerThreads", "8"); |
| 89 | + VortexBatchExec exec = new VortexBatchExec(List.of("/data/a.vortex"), COLUMNS, options, new Predicate[0]); |
| 90 | + |
| 91 | + VortexFilePartition partition = (VortexFilePartition) exec.planInputPartitions()[0]; |
| 92 | + |
| 93 | + assertEquals("8", partition.formatOptions().get("vortex.workerThreads")); |
| 94 | + } |
| 95 | +} |
0 commit comments