Fix pandas 3.0 compatibility: StringDtype, datetime resolution, deprecated APIs, pyspark.pandas import (#1527)

* Initial plan

* Fix pandas 3.0 compatibility: StringDtype, datetime resolution, deprecated APIs

- Add "str" to string dtype name checks in data.py, ts_data.py, generic_task.py
  (pandas 3.0 uses StringDtype(na_value=nan) with dtype.name="str")
- Wrap np.issubdtype(columns.dtype, np.integer) in try/except for StringDtype
  column indexes in pandas 3.0
- Use is_datetime64_any_dtype() instead of hardcoded "datetime64[ns]" check
  (pandas 3.0 uses varying datetime resolutions like datetime64[s], [ms], [us])
- Replace fillna(method="ffill") with ffill() in test files (removed in pandas 3.0)
- Replace deprecated frequency aliases "T"->"min", "H"->"h" in test files

Agent-Logs-Url: https://github.com/microsoft/FLAML/sessions/e2339d06-2236-4c1e-901b-b00a558f3796

Co-authored-by: thinkall <3197038+thinkall@users.noreply.github.com>

* CI: pin pandas 2 on ubuntu+python3.12 for backward compat testing

Agent-Logs-Url: https://github.com/microsoft/FLAML/sessions/7c3ce4f8-8623-4d3e-8986-2fa83e1daa5a

Co-authored-by: thinkall <3197038+thinkall@users.noreply.github.com>

* Add is_datetime64_any_dtype fallback in ts_data.py except ImportError block

Agent-Logs-Url: https://github.com/microsoft/FLAML/sessions/2b20aa86-81b1-4b3b-b6af-0cf7669c5575

Co-authored-by: thinkall <3197038+thinkall@users.noreply.github.com>

* Guard is_datetime64_any_dtype calls with None check in ts_data.py

Agent-Logs-Url: https://github.com/microsoft/FLAML/sessions/395c2eef-887f-4278-a4e0-4d4d276afeee

Co-authored-by: thinkall <3197038+thinkall@users.noreply.github.com>

* Fix Spark test failures: separate core pyspark imports from pyspark.pandas

Agent-Logs-Url: https://github.com/microsoft/FLAML/sessions/28e597b7-ae5e-41ba-957d-04aaa78834bd

Co-authored-by: thinkall <3197038+thinkall@users.noreply.github.com>

* Narrow pyspark.pandas fallback to ImportError instead of broad Exception

Agent-Logs-Url: https://github.com/microsoft/FLAML/sessions/28e597b7-ae5e-41ba-957d-04aaa78834bd

Co-authored-by: thinkall <3197038+thinkall@users.noreply.github.com>

* Guard set_option and ps.from_pandas calls in to_pandas_on_spark when pyspark.pandas unavailable

Agent-Logs-Url: https://github.com/microsoft/FLAML/sessions/60ca1746-e262-40e3-a970-2db40bda57dd

Co-authored-by: thinkall <3197038+thinkall@users.noreply.github.com>

* Pin pandas<3 for all pyspark environments and add pandas<3 to spark extra in setup.py

Agent-Logs-Url: https://github.com/microsoft/FLAML/sessions/0512ff2f-87ab-4394-b688-ea1416c26c6a

Co-authored-by: thinkall <3197038+thinkall@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: thinkall <3197038+thinkall@users.noreply.github.com>
Co-authored-by: Li Jiang <bnujli@gmail.com>

Author: Copilot

.github/workflows/python-package.yml

@@ -64,6 +64,10 @@ jobs:
           pip install -e .
           python -c "import flaml"
           pip install -e .[test]
+      - name: On Ubuntu with pyspark, pin pandas<3 (pyspark doesn't support pandas 3.0 yet)
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version != '3.10'
+        run: |
+          pip install "pandas>=2.0,<3"
       - name: On Ubuntu python 3.11, install pyspark 3.5.1
         if: matrix.python-version == '3.11' && matrix.os == 'ubuntu-latest'
         run: |

flaml/automl/data.py

@@ -16,6 +16,11 @@
 from flaml.automl.spark import DataFrame, F, Series, T, pd, ps, psDataFrame, psSeries
 from flaml.automl.training_log import training_log_reader
 
+try:
+    from pandas.api.types import is_datetime64_any_dtype
+except ImportError:
+    is_datetime64_any_dtype = None
+
 try:
     from scipy.sparse import issparse, vstack
 except ImportError:
@@ -302,7 +307,7 @@ def fit_transform(self, X: Union[DataFrame, np.ndarray], y, task: Union[str, "Ta
                     y = y.rename(TS_VALUE_COL)
             for column in X.columns:
                 # sklearn\utils\validation.py needs int/float values
-                if X[column].dtype.name in ("object", "category", "string"):
+                if X[column].dtype.name in ("object", "category", "string", "str"):
                     if X[column].nunique() == 1 or X[column].nunique(dropna=True) == n - X[column].isnull().sum():
                         X.drop(columns=column, inplace=True)
                         drop = True
@@ -318,7 +323,7 @@ def fit_transform(self, X: Union[DataFrame, np.ndarray], y, task: Union[str, "Ta
                     X.drop(columns=column, inplace=True)
                     drop = True
                 else:  # datetime or numeric
-                    if X[column].dtype.name == "datetime64[ns]":
+                    if is_datetime64_any_dtype is not None and is_datetime64_any_dtype(X[column]):
                         tmp_dt = X[column].dt
                         new_columns_dict = {
                             f"year_{column}": tmp_dt.year,
@@ -347,9 +352,11 @@ def fit_transform(self, X: Union[DataFrame, np.ndarray], y, task: Union[str, "Ta
                 X[cat_columns] = X[cat_columns].astype("category")
             if num_columns:
                 X_num = X[num_columns]
-                if np.issubdtype(X_num.columns.dtype, np.integer) and (
-                    drop or min(X_num.columns) != 0 or max(X_num.columns) != X_num.shape[1] - 1
-                ):
+                try:
+                    is_int_cols = np.issubdtype(X_num.columns.dtype, np.integer)
+                except TypeError:
+                    is_int_cols = False
+                if is_int_cols and (drop or min(X_num.columns) != 0 or max(X_num.columns) != X_num.shape[1] - 1):
                     X_num.columns = range(X_num.shape[1])
                     drop = True
                 else:
@@ -435,7 +442,7 @@ def transform(self, X: Union[DataFrame, np.array]):
             if self._task.is_ts_forecast():
                 X.insert(0, TS_TIMESTAMP_COL, ds_col)
             for column in cat_columns:
-                if X[column].dtype.name == "object":
+                if X[column].dtype.name in ("object", "string", "str"):
                     X[column] = X[column].fillna("__NAN__")
                 elif X[column].dtype.name == "category":
                     current_categories = X[column].cat.categories

flaml/automl/spark/__init__.py

@@ -5,12 +5,8 @@
 os.environ["PYARROW_IGNORE_TIMEZONE"] = "1"
 try:
     import pyspark
-    import pyspark.pandas as ps
     import pyspark.sql.functions as F
     import pyspark.sql.types as T
-    from pyspark.pandas import DataFrame as psDataFrame
-    from pyspark.pandas import Series as psSeries
-    from pyspark.pandas import set_option
     from pyspark.sql import DataFrame as sparkDataFrame
     from pyspark.sql import SparkSession
     from pyspark.util import VersionUtils
@@ -29,6 +25,20 @@ class psDataFrame:
 else:
     ERROR = None
     _spark_major_minor_version = VersionUtils.majorMinorVersion(pyspark.__version__)
+    # pyspark.pandas may fail with newer pandas versions (e.g., pandas 3.0)
+    # but core pyspark functionality should still work
+    try:
+        import pyspark.pandas as ps
+        from pyspark.pandas import DataFrame as psDataFrame
+        from pyspark.pandas import Series as psSeries
+        from pyspark.pandas import set_option
+    except ImportError:
+
+        class psDataFrame:
+            pass
+
+        ps = psSeries = psDataFrame
+        set_option = None
 
 try:
     import pandas as pd

flaml/automl/spark/utils.py

@@ -58,7 +58,8 @@ def to_pandas_on_spark(
     print(pss)
     ```
     """
-    set_option("compute.default_index_type", default_index_type)
+    if set_option is not None:
+        set_option("compute.default_index_type", default_index_type)
     try:
         orig_ps_conf = ps.get_option("compute.fail_on_ansi_mode")
     except Exception:
@@ -68,7 +69,14 @@ def to_pandas_on_spark(
 
     try:
         if isinstance(df, (DataFrame, Series)):
-            return ps.from_pandas(df)
+            if set_option is not None:
+                return ps.from_pandas(df)
+            else:
+                raise ImportError(
+                    "pyspark.pandas is not available (likely incompatible with installed pandas version). "
+                    "Cannot convert pandas DataFrame/Series to pandas-on-Spark. "
+                    "Consider downgrading pandas or upgrading pyspark."
+                )
         elif isinstance(df, sparkDataFrame):
             if _spark_major_minor_version[0] == 3 and _spark_major_minor_version[1] < 3:
                 return df.to_pandas_on_spark(index_col=index_col)

flaml/automl/task/generic_task.py

@@ -167,6 +167,7 @@ def validate_data(
                 assert X[column].dtype.name in (
                     "object",
                     "string",
+                    "str",
                 ), "If the task is an NLP task, X can only contain text columns"
                 for _, each_cell in X[column].items():
                     if each_cell is not None:

flaml/automl/time_series/ts_data.py

@@ -25,6 +25,7 @@ class PD:
     pd.DataFrame = None
     pd.Series = None
     DataFrame = Series = None
+    is_datetime64_any_dtype = None
 
 
 # dataclass will remove empty default value even with field(default_factory=lambda: [])
@@ -272,7 +273,7 @@ def enrich_dataframe(
 
     new_cols = []
     for col in df.columns:
-        if df[col].dtype.name == "datetime64[ns]":
+        if is_datetime64_any_dtype is not None and is_datetime64_any_dtype(df[col]):
             extras = monthly_fourier_features(df[col], fourier_degree)
             extras.columns = [f"{col}_{c}" for c in extras.columns]
             extras.index = df.index
@@ -403,12 +404,12 @@ def fit(self, X: Union[DataFrame, np.array], y):
                 continue
 
             # Robust datetime detection (covers datetime64[ms/us/ns], tz-aware, etc.)
-            if is_datetime64_any_dtype(X[column]):
+            if is_datetime64_any_dtype is not None and is_datetime64_any_dtype(X[column]):
                 self.datetime_columns.append(column)
                 continue
 
             # sklearn/utils/validation.py needs int/float values
-            if X[column].dtype.name in ("object", "category", "string"):
+            if X[column].dtype.name in ("object", "category", "string", "str"):
                 if (
                     # drop columns where all values are the same
                     X[column].nunique() == 1

setup.py

@@ -47,6 +47,7 @@
         ],
         "spark": [
             "pyspark>=3.2.0",
+            "pandas<3",
             "joblibspark>=0.5.0",
             "joblib<=1.3.2",
         ],

test/automl/test_extra_models.py

@@ -209,8 +209,8 @@ def load_multi_dataset():
     df["timeStamp"] = pd.to_datetime(df["timeStamp"])
     df = df.set_index("timeStamp")
     df = df.resample("D").mean()
-    df["temp"] = df["temp"].fillna(method="ffill")
-    df["precip"] = df["precip"].fillna(method="ffill")
+    df["temp"] = df["temp"].ffill()
+    df["precip"] = df["precip"].ffill()
     df = df[:-2]  # last two rows are NaN for 'demand' column so remove them
     df = df.reset_index()
 

test/automl/test_forecast.py

@@ -163,7 +163,7 @@ def test_numpy_large():
 
     from flaml import AutoML
 
-    X_train = pd.date_range("2017-01-01", periods=70000, freq="T")
+    X_train = pd.date_range("2017-01-01", periods=70000, freq="min")
     y_train = pd.DataFrame(np.random.randint(6500, 7500, 70000))
     automl = AutoML()
     automl.fit(
@@ -187,8 +187,8 @@ def load_multi_dataset():
     df["timeStamp"] = pd.to_datetime(df["timeStamp"])
     df = df.set_index("timeStamp")
     df = df.resample("D").mean()
-    df["temp"] = df["temp"].fillna(method="ffill")
-    df["precip"] = df["precip"].fillna(method="ffill")
+    df["temp"] = df["temp"].ffill()
+    df["precip"] = df["precip"].ffill()
     df = df[:-2]  # last two rows are NaN for 'demand' column so remove them
     df = df.reset_index()
 

test/automl/test_max_iter_1.py

@@ -6,7 +6,7 @@
 
 
 def test_max_iter_1():
-    date_rng = pd.date_range(start="2024-01-01", periods=100, freq="H")
+    date_rng = pd.date_range(start="2024-01-01", periods=100, freq="h")
     X = pd.DataFrame({"ds": date_rng})
     y_train_24h = np.random.rand(len(X)) * 100