Skip to content

Commit d54f70d

Browse files
jackylee-chjackylee-ch
authored
test(spark): characterize PartitionPathUtils partition discovery and materialization (#8783)
## Rationale for this change `PartitionPathUtils` (`java/vortex-spark/src/main/java/dev/vortex/spark/read/PartitionPathUtils.java`) discovers Hive-style partition columns from file paths and materializes them as constant column vectors on the read path. It currently has no test coverage, despite encoding several subtle behaviors: URL decoding of keys and values, the `__HIVE_DEFAULT_PARTITION__` null sentinel, first-`=` splitting, type inference precedence (int → long → double → boolean → string), and per-type parsing into `ConstantColumnVector`. This continues the characterization-test series from #8770 (read-path `ArrowUtils`) and #8782 (write-path `SparkToArrowSchema`). ## What changes are included in this PR? A new `PartitionPathUtilsTest` covering all three public methods: - **`parsePartitionValues`**: extracts ordered `key=value` segments from paths; ignores segments without the `key=value` shape (including empty keys or values); URL-decodes both keys and values (`New%20York` → `New York`); splits on the first `=` so values may themselves contain `=`. - **`inferPartitionColumnType`**: infers `Integer` for int-width values, `Long` for wider integrals, `Double` for decimal-point values, `Boolean` for case-insensitive true/false, and falls back to `String` (including for null and the `__HIVE_DEFAULT_PARTITION__` sentinel). - **`createConstantVector`**: null and the Hive default-partition sentinel produce null vectors; string/int/long/short/byte/boolean/float/double values are parsed into their target types; `DateType` parses as days-since-epoch (int) and both timestamp types as micros-since-epoch (long); unrecognized target types fall back to UTF8 strings. This is a **test-only** change; no production code is modified. 19 tests, all passing locally (`:vortex-spark_2.12:test --tests 'dev.vortex.spark.read.PartitionPathUtilsTest'`). ## What APIs are changed? Are there any user-facing changes? None. No production code or public API is touched. Signed-off-by: jackylee <qcsd2011@gmail.com>
1 parent 198cbd5 commit d54f70d

1 file changed

Lines changed: 199 additions & 0 deletions

File tree

‎java/vortex-spark/src/test/java/dev/vortex/spark/read/PartitionPathUtilsTest.java‎

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
package dev.vortex.spark.read;
5+
6+
import static org.junit.jupiter.api.Assertions.assertEquals;
7+
import static org.junit.jupiter.api.Assertions.assertFalse;
8+
import static org.junit.jupiter.api.Assertions.assertTrue;
9+
10+
import java.util.List;
11+
import java.util.Map;
12+
import org.apache.spark.sql.execution.vectorized.ConstantColumnVector;
13+
import org.apache.spark.sql.types.DataTypes;
14+
import org.apache.spark.unsafe.types.UTF8String;
15+
import org.junit.jupiter.api.DisplayName;
16+
import org.junit.jupiter.api.Test;
17+
18+
/**
19+
* Unit tests for {@link PartitionPathUtils}, which discovers and materializes Hive-style partition columns from file
20+
* paths.
21+
*
22+
* <p>Characterizes partition-value parsing (including URL decoding and the {@code __HIVE_DEFAULT_PARTITION__}
23+
* sentinel), partition column type inference, and constant-vector materialization for each supported type.
24+
*/
25+
final class PartitionPathUtilsTest {
26+
// --- parsePartitionValues ---
27+
28+
@Test
29+
@DisplayName("Parses key=value segments from a Hive-style partition path")
30+
void parsesKeyValueSegments() {
31+
Map<String, String> values =
32+
PartitionPathUtils.parsePartitionValues("/data/warehouse/year=2024/month=12/part-0.vortex");
33+
assertEquals(Map.of("year", "2024", "month", "12"), values);
34+
}
35+
36+
@Test
37+
@DisplayName("Preserves the order of partition segments in the path")
38+
void preservesSegmentOrder() {
39+
Map<String, String> values = PartitionPathUtils.parsePartitionValues("/tbl/b=2/a=1/c=3/file.vortex");
40+
assertEquals(List.of("b", "a", "c"), List.copyOf(values.keySet()));
41+
}
42+
43+
@Test
44+
@DisplayName("Ignores segments without key=value shape")
45+
void ignoresNonPartitionSegments() {
46+
Map<String, String> values = PartitionPathUtils.parsePartitionValues("/data/plain/dir/file.vortex");
47+
assertTrue(values.isEmpty());
48+
}
49+
50+
@Test
51+
@DisplayName("Ignores segments with an empty key or empty value")
52+
void ignoresEmptyKeyOrValue() {
53+
assertTrue(PartitionPathUtils.parsePartitionValues("/tbl/=v/file").isEmpty());
54+
assertTrue(PartitionPathUtils.parsePartitionValues("/tbl/k=/file").isEmpty());
55+
}
56+
57+
@Test
58+
@DisplayName("URL-decodes both keys and values")
59+
void urlDecodesKeysAndValues() {
60+
Map<String, String> values = PartitionPathUtils.parsePartitionValues("/tbl/city=New%20York/file.vortex");
61+
assertEquals(Map.of("city", "New York"), values);
62+
}
63+
64+
@Test
65+
@DisplayName("Splits on the first '=' so values may contain '='")
66+
void splitsOnFirstEquals() {
67+
Map<String, String> values = PartitionPathUtils.parsePartitionValues("/tbl/k=a=b/file.vortex");
68+
assertEquals(Map.of("k", "a=b"), values);
69+
}
70+
71+
// --- inferPartitionColumnType ---
72+
73+
@Test
74+
@DisplayName("Infers IntegerType for values that fit in an int")
75+
void infersInteger() {
76+
assertEquals(DataTypes.IntegerType, PartitionPathUtils.inferPartitionColumnType("42"));
77+
assertEquals(DataTypes.IntegerType, PartitionPathUtils.inferPartitionColumnType("-7"));
78+
}
79+
80+
@Test
81+
@DisplayName("Infers LongType for integral values wider than an int")
82+
void infersLong() {
83+
assertEquals(DataTypes.LongType, PartitionPathUtils.inferPartitionColumnType("9999999999"));
84+
}
85+
86+
@Test
87+
@DisplayName("Infers DoubleType for decimal-point values")
88+
void infersDouble() {
89+
assertEquals(DataTypes.DoubleType, PartitionPathUtils.inferPartitionColumnType("3.14"));
90+
}
91+
92+
@Test
93+
@DisplayName("Infers BooleanType for true/false in any case")
94+
void infersBoolean() {
95+
assertEquals(DataTypes.BooleanType, PartitionPathUtils.inferPartitionColumnType("true"));
96+
assertEquals(DataTypes.BooleanType, PartitionPathUtils.inferPartitionColumnType("FALSE"));
97+
}
98+
99+
@Test
100+
@DisplayName("Falls back to StringType for everything else")
101+
void fallsBackToString() {
102+
assertEquals(DataTypes.StringType, PartitionPathUtils.inferPartitionColumnType("hello"));
103+
assertEquals(DataTypes.StringType, PartitionPathUtils.inferPartitionColumnType("2024-12-01"));
104+
}
105+
106+
@Test
107+
@DisplayName("Null and __HIVE_DEFAULT_PARTITION__ infer StringType")
108+
void nullAndDefaultPartitionInferString() {
109+
assertEquals(DataTypes.StringType, PartitionPathUtils.inferPartitionColumnType(null));
110+
assertEquals(DataTypes.StringType, PartitionPathUtils.inferPartitionColumnType("__HIVE_DEFAULT_PARTITION__"));
111+
}
112+
113+
// --- createConstantVector ---
114+
115+
@Test
116+
@DisplayName("Null and __HIVE_DEFAULT_PARTITION__ materialize as null vectors")
117+
void nullValuesMaterializeAsNull() {
118+
ConstantColumnVector fromNull = PartitionPathUtils.createConstantVector(3, DataTypes.StringType, null);
119+
assertTrue(fromNull.isNullAt(0));
120+
121+
ConstantColumnVector fromSentinel =
122+
PartitionPathUtils.createConstantVector(3, DataTypes.StringType, "__HIVE_DEFAULT_PARTITION__");
123+
assertTrue(fromSentinel.isNullAt(0));
124+
}
125+
126+
@Test
127+
@DisplayName("String values materialize as UTF8 strings")
128+
void stringMaterializes() {
129+
ConstantColumnVector vec = PartitionPathUtils.createConstantVector(2, DataTypes.StringType, "us-east");
130+
assertFalse(vec.isNullAt(0));
131+
assertEquals(UTF8String.fromString("us-east"), vec.getUTF8String(0));
132+
}
133+
134+
@Test
135+
@DisplayName("Integral values materialize with the width of the target type")
136+
void integralTypesMaterialize() {
137+
assertEquals(
138+
42,
139+
PartitionPathUtils.createConstantVector(1, DataTypes.IntegerType, "42")
140+
.getInt(0));
141+
assertEquals(
142+
9999999999L,
143+
PartitionPathUtils.createConstantVector(1, DataTypes.LongType, "9999999999")
144+
.getLong(0));
145+
assertEquals(
146+
(short) 7,
147+
PartitionPathUtils.createConstantVector(1, DataTypes.ShortType, "7")
148+
.getShort(0));
149+
assertEquals(
150+
(byte) 3,
151+
PartitionPathUtils.createConstantVector(1, DataTypes.ByteType, "3")
152+
.getByte(0));
153+
}
154+
155+
@Test
156+
@DisplayName("DateType parses the value as days since epoch (int)")
157+
void dateMaterializesAsInt() {
158+
assertEquals(
159+
19000,
160+
PartitionPathUtils.createConstantVector(1, DataTypes.DateType, "19000")
161+
.getInt(0));
162+
}
163+
164+
@Test
165+
@DisplayName("Timestamp types parse the value as micros since epoch (long)")
166+
void timestampMaterializesAsLong() {
167+
assertEquals(
168+
1700000000000000L,
169+
PartitionPathUtils.createConstantVector(1, DataTypes.TimestampType, "1700000000000000")
170+
.getLong(0));
171+
assertEquals(
172+
1700000000000000L,
173+
PartitionPathUtils.createConstantVector(1, DataTypes.TimestampNTZType, "1700000000000000")
174+
.getLong(0));
175+
}
176+
177+
@Test
178+
@DisplayName("Boolean, float, and double values materialize with their target types")
179+
void booleanAndFloatingPointMaterialize() {
180+
assertTrue(PartitionPathUtils.createConstantVector(1, DataTypes.BooleanType, "true")
181+
.getBoolean(0));
182+
assertEquals(
183+
1.5f,
184+
PartitionPathUtils.createConstantVector(1, DataTypes.FloatType, "1.5")
185+
.getFloat(0));
186+
assertEquals(
187+
2.25,
188+
PartitionPathUtils.createConstantVector(1, DataTypes.DoubleType, "2.25")
189+
.getDouble(0));
190+
}
191+
192+
@Test
193+
@DisplayName("Unrecognized target types fall back to UTF8 strings")
194+
void unrecognizedTypeFallsBackToString() {
195+
ConstantColumnVector vec =
196+
PartitionPathUtils.createConstantVector(1, DataTypes.CalendarIntervalType, "whatever");
197+
assertEquals(UTF8String.fromString("whatever"), vec.getUTF8String(0));
198+
}
199+
}

0 commit comments

Comments
 (0)