Skip to content

Commit bda099a

Browse files
ueshindongjoon-hyun
ueshin
authored and
dongjoon-hyun
committed
[SPARK-56118][PS] Match pandas 3.0 bool handling in GroupBy.quantile
### What changes were proposed in this pull request? This PR updates pandas API on Spark `GroupBy.quantile` to align its bool-dtype behavior with pandas 3.0. Currently, `GroupBy.quantile` allows bool input and emits a `FutureWarning`. With this change, pandas API on Spark keeps that warning-based behavior when running against pandas versions earlier than 3.0, but raises `TypeError("Cannot use quantile with bool dtype")` when running with pandas 3.0 or later. The related groupby quantile tests were also updated to cover both version-dependent paths: - normal quantile behavior for numeric data - warning-compatible behavior for bool data on pandas < 3.0 - `TypeError` for bool data on pandas >= 3.0 - existing invalid `q` input checks ### Why are the changes needed? pandas 3.0 no longer allows `quantile` on bool dtype. pandas API on Spark should follow that behavior so groupby quantile results stay consistent with the pandas version it targets. Without this change, pandas API on Spark would continue accepting bool input under pandas 3.0 and diverge from pandas behavior. ### Does this PR introduce _any_ user-facing change? Yes, it will behave more like pandas 3. ### How was this patch tested? Updated the related tests. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #54929 from ueshin/issues/SPARK-56118/quantile. Authored-by: Takuya Ueshin <ueshin@databricks.com> Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
1 parent 7a81e1f commit bda099a

2 files changed

Lines changed: 77 additions & 46 deletions

File tree

‎python/pyspark/pandas/groupby.py‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -754,12 +754,15 @@ def quantile(self, q: float = 0.5, accuracy: int = 10000) -> FrameLike:
754754
if not 0 <= q <= 1:
755755
raise ValueError("'q' must be between 0 and 1. Got '%s' instead" % q)
756756
if any(isinstance(_agg_col.spark.data_type, BooleanType) for _agg_col in self._agg_columns):
757-
warnings.warn(
758-
f"Allowing bool dtype in {self.__class__.__name__}.quantile is deprecated "
759-
"and will raise in a future version, matching the Series/DataFrame behavior. "
760-
"Cast to uint8 dtype before calling quantile instead.",
761-
FutureWarning,
762-
)
757+
if LooseVersion(pd.__version__) < "3.0.0":
758+
warnings.warn(
759+
f"Allowing bool dtype in {self.__class__.__name__}.quantile is deprecated "
760+
"and will raise in a future version, matching the Series/DataFrame behavior. "
761+
"Cast to uint8 dtype before calling quantile instead.",
762+
FutureWarning,
763+
)
764+
else:
765+
raise TypeError("Cannot use quantile with bool dtype")
763766

764767
return self._reduce_for_stat_function(
765768
lambda col: F.percentile_approx(col.cast(DoubleType()), q, accuracy),

‎python/pyspark/pandas/tests/groupby/test_stat_adv.py‎

Lines changed: 68 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import numpy as np
1919
import pandas as pd
2020

21+
from pyspark.loose_version import LooseVersion
2122
from pyspark import pandas as ps
2223
from pyspark.testing.pandasutils import PandasOnSparkTestCase
2324
from pyspark.testing.sqlutils import SQLTestUtils
@@ -41,46 +42,73 @@ def psdf(self):
4142
return ps.from_pandas(self.pdf)
4243

4344
def test_quantile(self):
44-
dfs = [
45-
pd.DataFrame(
46-
[["a", 1], ["a", 2], ["a", 3], ["b", 1], ["b", 3], ["b", 5]], columns=["key", "val"]
47-
),
48-
pd.DataFrame(
49-
[["a", True], ["a", True], ["a", False], ["b", True], ["b", True], ["b", False]],
50-
columns=["key", "val"],
51-
),
52-
]
53-
for df in dfs:
54-
psdf = ps.from_pandas(df)
55-
# q accept float and int between 0 and 1
56-
for i in [0, 0.1, 0.5, 1]:
57-
self.assert_eq(
58-
df.groupby("key").quantile(q=i, interpolation="lower"),
59-
psdf.groupby("key").quantile(q=i),
60-
almost=True,
61-
)
62-
self.assert_eq(
63-
df.groupby("key")["val"].quantile(q=i, interpolation="lower"),
64-
psdf.groupby("key")["val"].quantile(q=i),
65-
almost=True,
66-
)
67-
# raise ValueError when q not in [0, 1]
68-
with self.assertRaises(ValueError):
69-
psdf.groupby("key").quantile(q=1.1)
70-
with self.assertRaises(ValueError):
71-
psdf.groupby("key").quantile(q=-0.1)
72-
with self.assertRaises(ValueError):
73-
psdf.groupby("key").quantile(q=2)
74-
with self.assertRaises(ValueError):
75-
psdf.groupby("key").quantile(q=np.nan)
76-
# raise TypeError when q type mismatch
77-
with self.assertRaises(TypeError):
78-
psdf.groupby("key").quantile(q="0.1")
79-
# raise NotImplementedError when q is list like type
80-
with self.assertRaises(NotImplementedError):
81-
psdf.groupby("key").quantile(q=(0.1, 0.5))
82-
with self.assertRaises(NotImplementedError):
83-
psdf.groupby("key").quantile(q=[0.1, 0.5])
45+
df = pd.DataFrame(
46+
[["a", 1], ["a", 2], ["a", 3], ["b", 1], ["b", 3], ["b", 5]], columns=["key", "val"]
47+
)
48+
boolean_df = pd.DataFrame(
49+
[["a", True], ["a", True], ["a", False], ["b", True], ["b", True], ["b", False]],
50+
columns=["key", "val"],
51+
)
52+
53+
if LooseVersion(pd.__version__) < "3.0.0":
54+
dfs = [df, boolean_df]
55+
error_dfs = []
56+
else:
57+
dfs = [df]
58+
error_dfs = [boolean_df]
59+
60+
for i, pdf in enumerate(dfs):
61+
with self.subTest(i=i):
62+
psdf = ps.from_pandas(pdf)
63+
# q accept float and int between 0 and 1
64+
for q in [0, 0.1, 0.5, 1]:
65+
with self.subTest(q=q):
66+
self.assert_eq(
67+
pdf.groupby("key").quantile(q=q, interpolation="lower"),
68+
psdf.groupby("key").quantile(q=q),
69+
almost=True,
70+
)
71+
self.assert_eq(
72+
pdf.groupby("key")["val"].quantile(q=q, interpolation="lower"),
73+
psdf.groupby("key")["val"].quantile(q=q),
74+
almost=True,
75+
)
76+
77+
# raise TypeError when bool dtype with pandas 3
78+
for i, pdf in enumerate(error_dfs):
79+
with self.subTest(i=i):
80+
psdf = ps.from_pandas(pdf)
81+
for q in [0, 0.1, 0.5, 1]:
82+
with self.subTest(q=q):
83+
with self.assertRaisesRegex(
84+
TypeError, "Cannot use quantile with bool dtype"
85+
):
86+
psdf.groupby("key").quantile(q=q)
87+
with self.assertRaisesRegex(
88+
TypeError, "Cannot use quantile with bool dtype"
89+
):
90+
psdf.groupby("key")["val"].quantile(q=q)
91+
92+
for i, pdf in enumerate([df, boolean_df]):
93+
with self.subTest(i=i):
94+
psdf = ps.from_pandas(pdf)
95+
# raise ValueError when q not in [0, 1]
96+
with self.assertRaises(ValueError):
97+
psdf.groupby("key").quantile(q=1.1)
98+
with self.assertRaises(ValueError):
99+
psdf.groupby("key").quantile(q=-0.1)
100+
with self.assertRaises(ValueError):
101+
psdf.groupby("key").quantile(q=2)
102+
with self.assertRaises(ValueError):
103+
psdf.groupby("key").quantile(q=np.nan)
104+
# raise TypeError when q type mismatch
105+
with self.assertRaises(TypeError):
106+
psdf.groupby("key").quantile(q="0.1")
107+
# raise NotImplementedError when q is list like type
108+
with self.assertRaises(NotImplementedError):
109+
psdf.groupby("key").quantile(q=(0.1, 0.5))
110+
with self.assertRaises(NotImplementedError):
111+
psdf.groupby("key").quantile(q=[0.1, 0.5])
84112

85113
def test_first(self):
86114
self._test_stat_func(lambda groupby_obj: groupby_obj.first())

0 commit comments

Comments
 (0)