Skip to content

Commit aa10a79

Browse files
zhengruifengzhengruifeng
committed
[SPARK-56179][PYTHON] Consolidate error classes for type mismatch - part 1
### What changes were proposed in this pull request? Consolidate error classes for mismatched types As the first step, consolidate the first 11 `NOT_xxx` ### Why are the changes needed? There are around 40 error classes for type mismatch ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? CI ### Was this patch authored or co-authored using generative AI tooling? Generated-by: Claude Code (claude-opus-4-6) Closes #54978 from zhengruifeng/consolidate_no_class_part1. Authored-by: Ruifeng Zheng <ruifengz@apache.org> Signed-off-by: Ruifeng Zheng <ruifengz@apache.org>
1 parent 992d932 commit aa10a79

20 files changed

Lines changed: 206 additions & 143 deletions

‎python/pyspark/errors/error-conditions.json‎

Lines changed: 6 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -602,72 +602,6 @@
602602
"Value for `<arg_name>` must be greater than or equal to 0, got '<arg_value>'."
603603
]
604604
},
605-
"NOT_BOOL": {
606-
"message": [
607-
"Argument `<arg_name>` should be a bool, got <arg_type>."
608-
],
609-
"sqlState": "42K09"
610-
},
611-
"NOT_BOOL_OR_DICT_OR_FLOAT_OR_INT_OR_LIST_OR_STR_OR_TUPLE": {
612-
"message": [
613-
"Argument `<arg_name>` should be a bool, dict, float, int, str or tuple, got <arg_type>."
614-
],
615-
"sqlState": "42K09"
616-
},
617-
"NOT_BOOL_OR_DICT_OR_FLOAT_OR_INT_OR_STR": {
618-
"message": [
619-
"Argument `<arg_name>` should be a bool, dict, float, int or str, got <arg_type>."
620-
],
621-
"sqlState": "42K09"
622-
},
623-
"NOT_BOOL_OR_FLOAT_OR_INT": {
624-
"message": [
625-
"Argument `<arg_name>` should be a bool, float or int, got <arg_type>."
626-
],
627-
"sqlState": "42K09"
628-
},
629-
"NOT_BOOL_OR_FLOAT_OR_INT_OR_LIST_OR_NONE_OR_STR_OR_TUPLE": {
630-
"message": [
631-
"Argument `<arg_name>` should be a bool, float, int, list, None, str or tuple, got <arg_type>."
632-
],
633-
"sqlState": "42K09"
634-
},
635-
"NOT_BOOL_OR_FLOAT_OR_INT_OR_STR": {
636-
"message": [
637-
"Argument `<arg_name>` should be a bool, float, int or str, got <arg_type>."
638-
],
639-
"sqlState": "42K09"
640-
},
641-
"NOT_BOOL_OR_STR": {
642-
"message": [
643-
"Argument `<arg_name>` should be a bool or str, got <arg_type>."
644-
],
645-
"sqlState": "42K09"
646-
},
647-
"NOT_CALLABLE": {
648-
"message": [
649-
"Argument `<arg_name>` should be a callable, got <arg_type>."
650-
],
651-
"sqlState": "42K09"
652-
},
653-
"NOT_COLUMN": {
654-
"message": [
655-
"Argument `<arg_name>` should be a Column, got <arg_type>."
656-
],
657-
"sqlState": "42K09"
658-
},
659-
"NOT_COLUMN_OR_DATATYPE_OR_STR": {
660-
"message": [
661-
"Argument `<arg_name>` should be a Column, str or DataType, but got <arg_type>."
662-
],
663-
"sqlState": "42K09"
664-
},
665-
"NOT_COLUMN_OR_FLOAT_OR_INT_OR_LIST_OR_STR": {
666-
"message": [
667-
"Argument `<arg_name>` should be a Column, float, integer, list or string, got <arg_type>."
668-
],
669-
"sqlState": "42K09"
670-
},
671605
"NOT_COLUMN_OR_INT": {
672606
"message": [
673607
"Argument `<arg_name>` should be a Column or int, got <arg_type>."
@@ -716,6 +650,12 @@
716650
],
717651
"sqlState": "42K09"
718652
},
653+
"NOT_EXPECTED_TYPE": {
654+
"message": [
655+
"Argument `<arg_name>` should be <expected_type>, got <arg_type>."
656+
],
657+
"sqlState": "42K09"
658+
},
719659
"NOT_EXPRESSION": {
720660
"message": [
721661
"Argument `<arg_name>` should be an Expression, got <arg_type>."

‎python/pyspark/pandas/utils.py‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -955,27 +955,43 @@ def spark_column_equals(left: Column, right: Column) -> bool:
955955

956956
if not isinstance(left, ConnectColumn):
957957
raise PySparkTypeError(
958-
errorClass="NOT_COLUMN",
959-
messageParameters={"arg_name": "left", "arg_type": type(left).__name__},
958+
errorClass="NOT_EXPECTED_TYPE",
959+
messageParameters={
960+
"expected_type": "Column",
961+
"arg_name": "left",
962+
"arg_type": type(left).__name__,
963+
},
960964
)
961965
if not isinstance(right, ConnectColumn):
962966
raise PySparkTypeError(
963-
errorClass="NOT_COLUMN",
964-
messageParameters={"arg_name": "right", "arg_type": type(right).__name__},
967+
errorClass="NOT_EXPECTED_TYPE",
968+
messageParameters={
969+
"expected_type": "Column",
970+
"arg_name": "right",
971+
"arg_type": type(right).__name__,
972+
},
965973
)
966974
return repr(left).replace("`", "") == repr(right).replace("`", "")
967975
else:
968976
from pyspark.sql.classic.column import Column as ClassicColumn
969977

970978
if not isinstance(left, ClassicColumn):
971979
raise PySparkTypeError(
972-
errorClass="NOT_COLUMN",
973-
messageParameters={"arg_name": "left", "arg_type": type(left).__name__},
980+
errorClass="NOT_EXPECTED_TYPE",
981+
messageParameters={
982+
"expected_type": "Column",
983+
"arg_name": "left",
984+
"arg_type": type(left).__name__,
985+
},
974986
)
975987
if not isinstance(right, ClassicColumn):
976988
raise PySparkTypeError(
977-
errorClass="NOT_COLUMN",
978-
messageParameters={"arg_name": "right", "arg_type": type(right).__name__},
989+
errorClass="NOT_EXPECTED_TYPE",
990+
messageParameters={
991+
"expected_type": "Column",
992+
"arg_name": "right",
993+
"arg_type": type(right).__name__,
994+
},
979995
)
980996
return left._jc.equals(right._jc)
981997

‎python/pyspark/sql/classic/column.py‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -381,8 +381,12 @@ def withField(self, fieldName: str, col: ParentColumn) -> ParentColumn:
381381

382382
if not isinstance(col, Column):
383383
raise PySparkTypeError(
384-
errorClass="NOT_COLUMN",
385-
messageParameters={"arg_name": "col", "arg_type": type(col).__name__},
384+
errorClass="NOT_EXPECTED_TYPE",
385+
messageParameters={
386+
"expected_type": "Column",
387+
"arg_name": "col",
388+
"arg_type": type(col).__name__,
389+
},
386390
)
387391

388392
return Column(self._jc.withField(fieldName, col._jc))
@@ -587,8 +591,12 @@ def between(
587591
def when(self, condition: ParentColumn, value: Any) -> ParentColumn:
588592
if not isinstance(condition, Column):
589593
raise PySparkTypeError(
590-
errorClass="NOT_COLUMN",
591-
messageParameters={"arg_name": "condition", "arg_type": type(condition).__name__},
594+
errorClass="NOT_EXPECTED_TYPE",
595+
messageParameters={
596+
"expected_type": "Column",
597+
"arg_name": "condition",
598+
"arg_type": type(condition).__name__,
599+
},
592600
)
593601
v = value._jc if isinstance(value, Column) else enum_to_value(value)
594602
jc = self._jc.when(condition._jc, v)

‎python/pyspark/sql/classic/dataframe.py‎

Lines changed: 32 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -252,8 +252,9 @@ def explain(
252252
if not (is_no_argument or is_extended_case or is_extended_as_mode or is_mode_case):
253253
if (extended is not None) and (not isinstance(extended, (bool, str))):
254254
raise PySparkTypeError(
255-
errorClass="NOT_BOOL_OR_STR",
255+
errorClass="NOT_EXPECTED_TYPE",
256256
messageParameters={
257+
"expected_type": "bool or str",
257258
"arg_name": "extended",
258259
"arg_type": type(extended).__name__,
259260
},
@@ -306,8 +307,12 @@ def _show_string(
306307

307308
if not isinstance(vertical, bool):
308309
raise PySparkTypeError(
309-
errorClass="NOT_BOOL",
310-
messageParameters={"arg_name": "vertical", "arg_type": type(vertical).__name__},
310+
errorClass="NOT_EXPECTED_TYPE",
311+
messageParameters={
312+
"expected_type": "bool",
313+
"arg_name": "vertical",
314+
"arg_type": type(vertical).__name__,
315+
},
311316
)
312317

313318
if isinstance(truncate, bool) and truncate:
@@ -317,8 +322,9 @@ def _show_string(
317322
int_truncate = int(truncate)
318323
except ValueError:
319324
raise PySparkTypeError(
320-
errorClass="NOT_BOOL",
325+
errorClass="NOT_EXPECTED_TYPE",
321326
messageParameters={
327+
"expected_type": "bool",
322328
"arg_name": "truncate",
323329
"arg_type": type(truncate).__name__,
324330
},
@@ -1013,8 +1019,12 @@ def __getitem__(
10131019
return Column(jc)
10141020
else:
10151021
raise PySparkTypeError(
1016-
errorClass="NOT_COLUMN_OR_FLOAT_OR_INT_OR_LIST_OR_STR",
1017-
messageParameters={"arg_name": "item", "arg_type": type(item).__name__},
1022+
errorClass="NOT_EXPECTED_TYPE",
1023+
messageParameters={
1024+
"expected_type": "Column, float, integer, list or string",
1025+
"arg_name": "item",
1026+
"arg_type": type(item).__name__,
1027+
},
10181028
)
10191029

10201030
def __getattr__(self, name: str) -> Column:
@@ -1285,8 +1295,12 @@ def fillna(
12851295
) -> ParentDataFrame:
12861296
if not isinstance(value, (float, int, str, bool, dict)):
12871297
raise PySparkTypeError(
1288-
errorClass="NOT_BOOL_OR_DICT_OR_FLOAT_OR_INT_OR_STR",
1289-
messageParameters={"arg_name": "value", "arg_type": type(value).__name__},
1298+
errorClass="NOT_EXPECTED_TYPE",
1299+
messageParameters={
1300+
"expected_type": "bool, dict, float, int or str",
1301+
"arg_name": "value",
1302+
"arg_type": type(value).__name__,
1303+
},
12901304
)
12911305

12921306
# Note that bool validates isinstance(int), but we don't want to
@@ -1384,8 +1398,9 @@ def all_of_(xs: Iterable) -> bool:
13841398
valid_types = (bool, float, int, str, list, tuple)
13851399
if not isinstance(to_replace, valid_types + (dict,)):
13861400
raise PySparkTypeError(
1387-
errorClass="NOT_BOOL_OR_DICT_OR_FLOAT_OR_INT_OR_LIST_OR_STR_OR_TUPLE",
1401+
errorClass="NOT_EXPECTED_TYPE",
13881402
messageParameters={
1403+
"expected_type": "bool, dict, float, int, str or tuple",
13891404
"arg_name": "to_replace",
13901405
"arg_type": type(to_replace).__name__,
13911406
},
@@ -1397,8 +1412,9 @@ def all_of_(xs: Iterable) -> bool:
13971412
and not isinstance(to_replace, dict)
13981413
):
13991414
raise PySparkTypeError(
1400-
errorClass="NOT_BOOL_OR_FLOAT_OR_INT_OR_LIST_OR_NONE_OR_STR_OR_TUPLE",
1415+
errorClass="NOT_EXPECTED_TYPE",
14011416
messageParameters={
1417+
"expected_type": "bool, float, int, list, None, str or tuple",
14021418
"arg_name": "value",
14031419
"arg_type": type(value).__name__,
14041420
},
@@ -1648,8 +1664,12 @@ def withColumns(self, *colsMap: Dict[str, Column]) -> ParentDataFrame:
16481664
def withColumn(self, colName: str, col: Column) -> ParentDataFrame:
16491665
if not isinstance(col, Column):
16501666
raise PySparkTypeError(
1651-
errorClass="NOT_COLUMN",
1652-
messageParameters={"arg_name": "col", "arg_type": type(col).__name__},
1667+
errorClass="NOT_EXPECTED_TYPE",
1668+
messageParameters={
1669+
"expected_type": "Column",
1670+
"arg_name": "col",
1671+
"arg_type": type(col).__name__,
1672+
},
16531673
)
16541674
return DataFrame(self._jdf.withColumn(colName, col._jc), self.sparkSession)
16551675

‎python/pyspark/sql/connect/column.py‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -301,8 +301,12 @@ def endswith(
301301
def when(self, condition: ParentColumn, value: Any) -> ParentColumn:
302302
if not isinstance(condition, Column):
303303
raise PySparkTypeError(
304-
errorClass="NOT_COLUMN",
305-
messageParameters={"arg_name": "condition", "arg_type": type(condition).__name__},
304+
errorClass="NOT_EXPECTED_TYPE",
305+
messageParameters={
306+
"expected_type": "Column",
307+
"arg_name": "condition",
308+
"arg_type": type(condition).__name__,
309+
},
306310
)
307311

308312
if not isinstance(self._expr, CaseWhen):
@@ -527,8 +531,12 @@ def withField(self, fieldName: str, col: ParentColumn) -> ParentColumn:
527531

528532
if not isinstance(col, Column):
529533
raise PySparkTypeError(
530-
errorClass="NOT_COLUMN",
531-
messageParameters={"arg_name": "col", "arg_type": type(col).__name__},
534+
errorClass="NOT_EXPECTED_TYPE",
535+
messageParameters={
536+
"expected_type": "Column",
537+
"arg_name": "col",
538+
"arg_type": type(col).__name__,
539+
},
532540
)
533541

534542
return Column(WithField(self._expr, fieldName, col._expr))

0 commit comments

Comments
 (0)