Skip to content

Commit 76202cd

Browse files
committed
test(spark): characterize VortexBatchExec input partition planning
1 parent b7b01d3 commit 76202cd

1 file changed

Lines changed: 95 additions & 0 deletions

File tree

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
package dev.vortex.spark.read;
5+
6+
import static org.junit.jupiter.api.Assertions.assertEquals;
7+
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
8+
9+
import dev.vortex.spark.VortexFilePartition;
10+
import java.util.List;
11+
import java.util.Map;
12+
import org.apache.spark.sql.connector.catalog.Column;
13+
import org.apache.spark.sql.connector.expressions.filter.Predicate;
14+
import org.apache.spark.sql.connector.read.InputPartition;
15+
import org.junit.jupiter.api.DisplayName;
16+
import org.junit.jupiter.api.Test;
17+
18+
/**
19+
* Unit tests for {@link VortexBatchExec} input-partition planning.
20+
*
21+
* <p>Characterizes how explicit {@code .vortex} paths are planned: one {@link VortexFilePartition} per file, each
22+
* carrying the requested read schema and the Hive-style partition values parsed from its own path. All paths used here
23+
* end in {@code .vortex} so planning never lists directories through native I/O.
24+
*/
25+
final class VortexBatchExecTest {
26+
27+
private static final List<Column> COLUMNS = List.of(
28+
Column.create("id", org.apache.spark.sql.types.DataTypes.IntegerType),
29+
Column.create("name", org.apache.spark.sql.types.DataTypes.StringType));
30+
31+
private static VortexBatchExec execFor(List<String> paths) {
32+
return new VortexBatchExec(paths, COLUMNS, Map.of(), new Predicate[0]);
33+
}
34+
35+
@Test
36+
@DisplayName("Plans one input partition per .vortex file")
37+
void plansOnePartitionPerFile() {
38+
VortexBatchExec exec = execFor(List.of("/data/a.vortex", "/data/b.vortex", "/data/c.vortex"));
39+
40+
InputPartition[] partitions = exec.planInputPartitions();
41+
42+
assertEquals(3, partitions.length);
43+
for (InputPartition partition : partitions) {
44+
assertInstanceOf(VortexFilePartition.class, partition);
45+
assertEquals(1, ((VortexFilePartition) partition).paths().size());
46+
}
47+
}
48+
49+
@Test
50+
@DisplayName("Each partition carries exactly its own file path, in input order")
51+
void partitionsKeepInputOrder() {
52+
VortexBatchExec exec = execFor(List.of("/data/first.vortex", "/data/second.vortex"));
53+
54+
InputPartition[] partitions = exec.planInputPartitions();
55+
56+
assertEquals(List.of("/data/first.vortex"), ((VortexFilePartition) partitions[0]).paths());
57+
assertEquals(List.of("/data/second.vortex"), ((VortexFilePartition) partitions[1]).paths());
58+
}
59+
60+
@Test
61+
@DisplayName("Partitions carry the requested read schema")
62+
void partitionsCarryReadSchema() {
63+
VortexBatchExec exec = execFor(List.of("/data/a.vortex"));
64+
65+
VortexFilePartition partition = (VortexFilePartition) exec.planInputPartitions()[0];
66+
67+
assertEquals(2, partition.readSchema().size());
68+
assertEquals("id", partition.readSchema().fields()[0].name());
69+
assertEquals("name", partition.readSchema().fields()[1].name());
70+
}
71+
72+
@Test
73+
@DisplayName("Hive-style partition values are parsed from each file's own path")
74+
void partitionValuesParsedPerFile() {
75+
VortexBatchExec exec = execFor(
76+
List.of("/tbl/year=2024/month=01/a.vortex", "/tbl/year=2025/month=02/b.vortex", "/tbl/plain.vortex"));
77+
78+
InputPartition[] partitions = exec.planInputPartitions();
79+
80+
assertEquals(Map.of("year", "2024", "month", "01"), ((VortexFilePartition) partitions[0]).partitionValues());
81+
assertEquals(Map.of("year", "2025", "month", "02"), ((VortexFilePartition) partitions[1]).partitionValues());
82+
assertEquals(Map.of(), ((VortexFilePartition) partitions[2]).partitionValues());
83+
}
84+
85+
@Test
86+
@DisplayName("Format options are propagated to every partition")
87+
void formatOptionsPropagated() {
88+
Map<String, String> options = Map.of("vortex.workerThreads", "8");
89+
VortexBatchExec exec = new VortexBatchExec(List.of("/data/a.vortex"), COLUMNS, options, new Predicate[0]);
90+
91+
VortexFilePartition partition = (VortexFilePartition) exec.planInputPartitions()[0];
92+
93+
assertEquals("8", partition.formatOptions().get("vortex.workerThreads"));
94+
}
95+
}

0 commit comments

Comments
 (0)