Compatibility for cuML deprecation warnings (rapidsai#20884)

This fixes some deprecated usage of cuml. It also updates our call to pytest to error on warnings, which will help us catch these warnings earlier.

This revealed a resource warning from an unclosed file handle.

Authors:
  - Tom Augspurger (https://github.com/TomAugspurger)

Approvers:
  - Bradley Dice (https://github.com/bdice)

URL: rapidsai#20884

Author: TomAugspurger

ci/cudf_pandas_scripts/third-party-integration/run-library-tests.sh

@@ -21,6 +21,7 @@ runtest() {
     pytest \
     "${plugin[@]}" \
     -v \
+    -W error \
     --continue-on-collection-errors \
     --cache-clear \
     --numprocesses="${NUM_PROCESSES}" \
@@ -40,6 +41,7 @@ main() {
     --compare \
     -p cudf.pandas \
     -v \
+    -W error \
     --continue-on-collection-errors \
     --cache-clear \
     --numprocesses="${NUM_PROCESSES}" \

python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/conftest.py

@@ -96,6 +96,24 @@ def read_all_results(pattern):
 
 
 def pytest_configure(config: _pytest.config.Config):
+    # Register custom markers to avoid PytestUnknownMarkWarning
+    config.addinivalue_line(
+        "markers",
+        "assert_eq(fn): custom assertion function for comparing results",
+    )
+    config.addinivalue_line(
+        "markers",
+        "xfail_gold: mark test as expected to fail in gold (pandas) pass",
+    )
+    config.addinivalue_line(
+        "markers",
+        "xfail_cudf_pandas: mark test as expected to fail in cudf.pandas pass",
+    )
+    config.addinivalue_line(
+        "markers",
+        "xfail_compare: mark test as expected to fail in compare pass",
+    )
+
     gold_basename = "results-gold"
     cudf_basename = "results-cudf-pandas"
     test_folder = os.path.join(os.path.dirname(__file__))
@@ -189,4 +207,5 @@ def pytest_unconfigure(config):
                     config.stash[file_handle_key].write(f.read())
                 os.remove(worker_result)
     # Close our file
+    config.stash[file_handle_key].close()
     del config.stash[file_handle_key]

python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_cugraph.py

@@ -11,8 +11,8 @@
     "betweenness_centrality",
     "degree_centrality",
     "katz_centrality",
-    "sorensen_coefficient",
-    "jaccard_coefficient",
+    "sorensen",
+    "jaccard",
 ]
 
 nx_algos = [
@@ -33,7 +33,13 @@ def assert_cugraph_equal(expect, got):
         assert expect == got
 
 
-pytestmark = pytest.mark.assert_eq(fn=assert_cugraph_equal)
+pytestmark = [
+    pytest.mark.assert_eq(fn=assert_cugraph_equal),
+    # We can't pass a valid value here to avoid the warning, so we ignore it.
+    pytest.mark.filterwarnings(
+        "ignore:This parameter is deprecated:PendingDeprecationWarning"
+    ),
+]
 
 
 @pytest.fixture(scope="session")
@@ -57,13 +63,15 @@ def adjacency_matrix():
 def test_cugraph_from_pandas_edgelist(df, algo):
     G = cugraph.Graph()
     G.from_pandas_edgelist(df)
+    G.store_transposed = algo == "katz_centrality"
     return getattr(cugraph, algo)(G).to_pandas().values
 
 
 @pytest.mark.parametrize("algo", cugraph_algos)
 def test_cugraph_from_pandas_adjacency(adjacency_matrix, algo):
     G = cugraph.Graph()
     G.from_pandas_adjacency(adjacency_matrix)
+    G.store_transposed = algo == "katz_centrality"
     res = getattr(cugraph, algo)(G).to_pandas()
     return res.sort_values(list(res.columns)).values
 
@@ -72,6 +80,7 @@ def test_cugraph_from_pandas_adjacency(adjacency_matrix, algo):
 def test_cugraph_from_numpy_array(df, algo):
     G = cugraph.Graph()
     G.from_numpy_array(df.values)
+    G.store_transposed = algo == "katz_centrality"
     return getattr(cugraph, algo)(G).to_pandas().values
 
 

python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_cuml.py

@@ -94,7 +94,7 @@ def test_random_forest(binary_classification_data):
     X_train, X_test, y_train, y_test = train_test_split(
         X, y, test_size=0.2, random_state=42
     )
-    model = RandomForestClassifier(n_estimators=100)
+    model = RandomForestClassifier(n_estimators=100, n_bins=len(X_train))
     model.fit(X_train, y_train)
     preds = model.predict(X_test)
     return preds.values

python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_numpy.py

@@ -31,7 +31,7 @@ def df():
 
 @pytest.mark.parametrize("op", reductions)
 def test_numpy_dataframe_reductions(df, op):
-    return getattr(np, op)(df)
+    return getattr(np, op)(df, axis=0)
 
 
 def test_numpy_dot(df):

python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_seaborn.py

@@ -3,31 +3,11 @@
 import pandas as pd
 import pytest
 import seaborn as sns
-from matplotlib.axes import Axes
-from matplotlib.collections import PathCollection
-from matplotlib.lines import Line2D
-from matplotlib.patches import Rectangle
-from pandas._testing import assert_equal
 
 
 def assert_plots_equal(expect, got):
-    if isinstance(expect, Axes) and isinstance(got, Axes):
-        for expect_ch, got_ch in zip(
-            expect.get_children(), got.get_children(), strict=True
-        ):
-            assert type(expect_ch) is type(got_ch)
-            if isinstance(expect_ch, Line2D):
-                assert_equal(expect_ch.get_xdata(), got_ch.get_xdata())
-                assert_equal(expect_ch.get_ydata(), got_ch.get_ydata())
-            elif isinstance(expect_ch, Rectangle):
-                assert expect_ch.get_height() == got_ch.get_height()
-    elif isinstance(expect, PathCollection) and isinstance(
-        got, PathCollection
-    ):
-        assert_equal(expect.get_offsets()[:, 0], got.get_offsets()[:, 0])
-        assert_equal(expect.get_offsets()[:, 1], got.get_offsets()[:, 1])
-    else:
-        assert_equal(expect, got)
+    # these are the coordinates of the matplotlib objects.
+    assert expect == got
 
 
 pytestmark = pytest.mark.assert_eq(fn=assert_plots_equal)
@@ -47,15 +27,20 @@ def df():
 
 def test_bar(df):
     ax = sns.barplot(data=df, x="x", y="y")
-    return ax
+    return [x.get_height().item() for x in ax.patches]
 
 
 def test_scatter(df):
     ax = sns.scatterplot(data=df, x="x", y="y", hue="hue")
-    return ax
+    assert len(ax.collections) == 1
+    paths = ax.collections[0].get_paths()
+    assert len(paths) == 1
+    return paths[0].vertices.tolist()
 
 
 def test_lineplot_with_sns_data():
     df = sns.load_dataset("flights")
-    ax = sns.lineplot(data=df, x="month", y="passengers")
-    return ax
+    ax = sns.lineplot(data=df, x="month", y="passengers", seed=0)
+    paths = ax.collections[0].get_paths()
+    assert len(paths) == 1
+    return paths[0].vertices.tolist()

python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_stumpy.py

@@ -30,6 +30,9 @@ def test_1d_time_series():
     return stumpy.stump(ts, m)
 
 
+@pytest.mark.filterwarnings(
+    "ignore::numba.core.errors.NumbaPerformanceWarning"
+)
 def test_1d_gpu():
     rng = np.random.default_rng(42)
     your_time_series = rng.random(10000)

python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_stumpy_distributed.py

@@ -1,6 +1,9 @@
 # SPDX-FileCopyrightText: Copyright (c) 2023-2025, NVIDIA CORPORATION.
 # SPDX-License-Identifier: Apache-2.0
 
+import os
+import warnings
+
 import numpy as np
 import pandas as pd
 import pytest
@@ -25,12 +28,30 @@ def as_float64(x):
 
 # Shared dask client for all tests in this module
 @pytest.fixture(scope="module")
-def dask_client():
-    with LocalCluster(n_workers=4, threads_per_worker=1) as cluster:
-        with Client(cluster) as dask_client:
-            yield dask_client
+def dask_client(worker_id: str):
+    worker_count = int(os.environ.get("PYTEST_XDIST_WORKER_COUNT", "0"))
+    if worker_count > 0:
+        # Avoid port conflicts with multiple test runners
+        worker_index = int(worker_id.removeprefix("gw"))
+        scheduler_port = 8800 + worker_index
+        dashboard_address = 8900 + worker_index
+    else:
+        scheduler_port = None
+        dashboard_address = None
+
+    with warnings.catch_warnings():
+        warnings.filterwarnings("ignore", category=Warning, message="Port")
+
+        with LocalCluster(
+            n_workers=1,
+            scheduler_port=scheduler_port,
+            dashboard_address=dashboard_address,
+        ) as cluster:
+            with Client(cluster) as client:
+                yield client
 
 
+@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning")
 def test_1d_distributed(dask_client):
     rng = np.random.default_rng(seed=42)
     ts = pd.Series(rng.random(100))

python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_tensorflow.py

@@ -1,10 +1,17 @@
 # SPDX-FileCopyrightText: Copyright (c) 2023-2025, NVIDIA CORPORATION.
 # SPDX-License-Identifier: Apache-2.0
 
+import warnings
+
 import numpy as np
 import pandas as pd
 import pytest
-import tensorflow as tf
+
+with warnings.catch_warnings():
+    # Silence a warning from numpy on keras import
+    warnings.filterwarnings("ignore", category=FutureWarning)
+    import tensorflow as tf
+
 
 SHUFFLE_BUFFER = 500
 BATCH_SIZE = 2

python/cudf/cudf_pandas_tests/third_party_integration_tests/tests/test_xgboost.py

@@ -102,6 +102,7 @@ def test_with_iter_quantile_dmatrix(
 
 
 @pytest.mark.parametrize("device", ["cpu", "cuda"])
+@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning")
 def test_with_external_memory(
     device: str,
     reg_batches_data: tuple[list[pd.DataFrame], list[pd.DataFrame]],
@@ -115,6 +116,7 @@ def test_with_external_memory(
 
 
 @pytest.mark.parametrize("device", ["cpu", "cuda"])
+@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning")
 def test_predict(device: str) -> np.ndarray:
     reg = xgb.XGBRegressor(n_estimators=2, device=device)
     X, y = make_regression(n_samples, n_features, random_state=11)