Skip to content

Commit da76f5b

Browse files
authored
fix(spark): reject MapType instead of emitting a childless Arrow List (#8811)
`SparkToArrowSchema` mapped `MapType` to a childless `ArrowType.List`, which is an invalid Arrow list (a List must have exactly one child). Drop the branch so `MapType` falls through to the existing fail-fast `UnsupportedOperationException`, and add a test. Signed-off-by: jackylee <qcsd2011@gmail.com>
1 parent 60c1227 commit da76f5b

2 files changed

Lines changed: 10 additions & 4 deletions

File tree

java/vortex-spark/src/main/java/dev/vortex/spark/write/SparkToArrowSchema.java

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import org.apache.spark.sql.types.FloatType;
2424
import org.apache.spark.sql.types.IntegerType;
2525
import org.apache.spark.sql.types.LongType;
26-
import org.apache.spark.sql.types.MapType;
2726
import org.apache.spark.sql.types.ShortType;
2827
import org.apache.spark.sql.types.StringType;
2928
import org.apache.spark.sql.types.StructField;
@@ -125,9 +124,6 @@ private static ArrowType convertType(DataType sparkType) {
125124
return new ArrowType.List();
126125
} else if (sparkType instanceof StructType) {
127126
return new ArrowType.Struct();
128-
} else if (sparkType instanceof MapType) {
129-
// Map is represented as List<Struct<key, value>> in Arrow
130-
return new ArrowType.List();
131127
} else {
132128
throw new UnsupportedOperationException("Unsupported Spark type for Arrow conversion: "
133129
+ sparkType.getClass().getName());

java/vortex-spark/src/test/java/dev/vortex/spark/write/SparkToArrowSchemaTest.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,4 +173,14 @@ void unsupportedTypeIsRejected() {
173173
assertThrows(UnsupportedOperationException.class, () -> SparkToArrowSchema.convert(schema));
174174
assertTrue(e.getMessage().contains("CalendarIntervalType"));
175175
}
176+
177+
@Test
178+
@DisplayName("MapType is rejected rather than converted to a childless List")
179+
void mapTypeIsRejected() {
180+
StructType schema =
181+
new StructType().add("m", DataTypes.createMapType(DataTypes.StringType, DataTypes.IntegerType));
182+
UnsupportedOperationException e =
183+
assertThrows(UnsupportedOperationException.class, () -> SparkToArrowSchema.convert(schema));
184+
assertTrue(e.getMessage().contains("MapType"));
185+
}
176186
}

0 commit comments

Comments
 (0)