Skip to content

Commit 94dd273

Browse files
committed
fix(spark): include the offending path when schema inference finds no Vortex files
1 parent b7b01d3 commit 94dd273

2 files changed

Lines changed: 73 additions & 2 deletions

File tree

java/vortex-spark/src/main/java/dev/vortex/spark/VortexDataSourceV2.java

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ public VortexDataSourceV2() {
6262
*
6363
* @param options the data source options containing file paths
6464
* @return the inferred Spark SQL schema
65-
* @throws RuntimeException if required path options are missing
65+
* @throws IllegalArgumentException if no Vortex files can be found under the supplied paths
6666
* @throws RuntimeException if there's an error reading the file or converting the schema
6767
*/
6868
@Override
@@ -87,7 +87,11 @@ public StructType inferSchema(CaseInsensitiveStringMap options) {
8787
.findFirst();
8888

8989
if (firstFile.isEmpty()) {
90-
throw new RuntimeException(String.format("UNABLE_TO_INFER_SCHEMA format: %s", shortName()));
90+
throw new IllegalArgumentException(String.format(
91+
"Unable to infer schema for %s: no .vortex files found under path %s. "
92+
+ "Check that the path is correct and contains at least one Vortex file, "
93+
+ "or provide an explicit schema.",
94+
shortName(), pathToInfer));
9195
} else {
9296
pathToInfer = firstFile.get();
9397
}
Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
package dev.vortex.spark;
5+
6+
import static org.junit.jupiter.api.Assertions.assertThrows;
7+
import static org.junit.jupiter.api.Assertions.assertTrue;
8+
9+
import java.nio.file.Path;
10+
import org.apache.spark.sql.SparkSession;
11+
import org.junit.jupiter.api.AfterAll;
12+
import org.junit.jupiter.api.BeforeAll;
13+
import org.junit.jupiter.api.DisplayName;
14+
import org.junit.jupiter.api.Test;
15+
import org.junit.jupiter.api.TestInstance;
16+
import org.junit.jupiter.api.io.TempDir;
17+
18+
/** Tests for schema inference failure reporting in {@link VortexDataSourceV2}. */
19+
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
20+
public final class VortexDataSourceInferSchemaTest {
21+
22+
private SparkSession spark;
23+
24+
@TempDir
25+
Path tempDir;
26+
27+
@BeforeAll
28+
public void setUp() {
29+
spark = SparkSession.builder()
30+
.appName("VortexInferSchemaTest")
31+
.master("local[1]")
32+
.config("spark.driver.host", "127.0.0.1")
33+
.config("spark.ui.enabled", "false")
34+
.getOrCreate();
35+
}
36+
37+
@AfterAll
38+
public void tearDown() {
39+
if (spark != null) {
40+
spark.stop();
41+
}
42+
}
43+
44+
@Test
45+
@DisplayName("Reading a directory without Vortex files reports the offending path")
46+
public void inferSchemaFailureNamesThePath() {
47+
String emptyDir = tempDir.toString();
48+
49+
Throwable thrown = assertThrows(
50+
Throwable.class, () -> spark.read().format("vortex").load(emptyDir));
51+
52+
String allMessages = messagesOf(thrown);
53+
assertTrue(
54+
allMessages.contains("no .vortex files found"),
55+
"error should explain that no Vortex files were found, got: " + allMessages);
56+
assertTrue(allMessages.contains(emptyDir), "error should name the offending path, got: " + allMessages);
57+
}
58+
59+
/** Concatenates the messages of the whole cause chain, since Spark may wrap data source exceptions. */
60+
private static String messagesOf(Throwable thrown) {
61+
StringBuilder sb = new StringBuilder();
62+
for (Throwable t = thrown; t != null; t = t.getCause()) {
63+
sb.append(t.getMessage()).append('\n');
64+
}
65+
return sb.toString();
66+
}
67+
}

0 commit comments

Comments
 (0)