Merge branch 'main' into yuwang-fix-try-cast

Author: sfc-gh-yuwang

CHANGELOG.md

@@ -7,6 +7,13 @@
 #### Bug Fixes
 
 - Fixed a bug where CTE optimization incorrectly deduplicated subtrees containing non-deterministic data generation functions (e.g. `uuid_string()`). 
+- Fixed a bug where vectorized UDFs using non-anaconda package repositories did not specify the pandas package by default.
+
+### Snowpark pandas API Updates
+
+#### Dependency Updates
+
+- Updated the supported `pandas` versions to <=2.4 (was previously <=2.3.1).
 
 ## 1.49.0 (TBD)
 

setup.py

@@ -43,7 +43,7 @@
 MODIN_REQUIREMENTS = [
     *PANDAS_REQUIREMENTS,
     f"modin{MODIN_DEPENDENCY_VERSION}",
-    "pandas<=2.3.1",
+    "pandas<=2.4",
     "tqdm",  # For progress bars during backend switching
     "ipywidgets",  # For enhanced progress bars in Jupyter notebooks
 ]

src/snowflake/snowpark/_internal/udf_utils.py

@@ -1253,6 +1253,7 @@ def resolve_imports_and_packages(
                     [],
                     artifact_repository=artifact_repository,
                     existing_packages_dict=existing_packages_dict,
+                    include_pandas=is_pandas_udf,
                 )
             )
         elif packages:

src/snowflake/snowpark/functions.py

@@ -11945,7 +11945,7 @@ def regr_sxy(y: ColumnOrName, x: ColumnOrName, _emit_ast: bool = True) -> Column
 
         >>> df = session.create_dataframe([[10, 11], [20, 22], [25, None], [30, 35]], schema=["v", "v2"])
         >>> df = df.filter(df["v2"].is_not_null())
-        >>> df.group_by("v").agg(regr_sxy(df["v"], df["v2"]).alias("regr_sxy")).collect()
+        >>> df.group_by("v").agg(regr_sxy(df["v"], df["v2"]).alias("regr_sxy")).sort("v").collect()
         [Row(V=10, REGR_SXY=0.0), Row(V=20, REGR_SXY=0.0), Row(V=30, REGR_SXY=0.0)]
     """
     y_col = _to_col_if_str(y, "regr_sxy")

tests/integ/modin/frame/test_select_dtypes.py

@@ -137,36 +137,47 @@ def test_select_dtypes_duplicate_col_names(include, exclude):
     )
 
 
-@pytest.mark.parametrize(
-    "include, exclude, exc, exc_match",
-    [
-        ([], [], ValueError, "at least one of include or exclude must be nonempty"),
-        (None, None, ValueError, "at least one of include or exclude must be nonempty"),
-        # python `int` is equivalent to any np.int dtype, but it is fine for an type in
-        # `include` to be a strict subtype of a type in `exclude` or vice versa
-        (int, int, ValueError, "include and exclude overlap"),
-        ([int], ["O", int], ValueError, "include and exclude overlap"),
-        (["O", int], [int], ValueError, "include and exclude overlap"),
-        (int, np.int32, ValueError, "include and exclude overlap"),
-        (int, np.int64, ValueError, "include and exclude overlap"),
-        ("datetime", np.datetime64, ValueError, "include and exclude overlap"),
-        ("O", object, ValueError, "include and exclude overlap"),
-        # string dtypes are prohibited by pandas
-        (str, None, TypeError, "string dtypes are not allowed, use 'object' instead"),
-        (None, str, TypeError, "string dtypes are not allowed, use 'object' instead"),
-        (
-            "timedelta64[s]",
-            None,
-            ValueError,
-            "'timedelta64[s]' is too specific of a frequency, try passing 'timedelta64'",
+# Error message differs slightly across pandas patch versions
+STR_DTYPE_ERROR = r"string dtypes are not allowed, use ('str' or )?'object' instead"
+
+SELECT_DTYPES_INVALID_PARAMS = [
+    ([], [], ValueError, "at least one of include or exclude must be nonempty"),
+    (None, None, ValueError, "at least one of include or exclude must be nonempty"),
+    # python `int` is equivalent to any np.int dtype, but it is fine for an type in
+    # `include` to be a strict subtype of a type in `exclude` or vice versa
+    (int, int, ValueError, "include and exclude overlap"),
+    ([int], ["O", int], ValueError, "include and exclude overlap"),
+    (["O", int], [int], ValueError, "include and exclude overlap"),
+    (int, np.int32, ValueError, "include and exclude overlap"),
+    (int, np.int64, ValueError, "include and exclude overlap"),
+    ("datetime", np.datetime64, ValueError, "include and exclude overlap"),
+    ("O", object, ValueError, "include and exclude overlap"),
+    # string dtypes are prohibited by pandas
+    (str, None, TypeError, STR_DTYPE_ERROR),
+    (None, str, TypeError, STR_DTYPE_ERROR),
+    (
+        "timedelta64[s]",
+        None,
+        ValueError,
+        re.escape(
+            "'timedelta64[s]' is too specific of a frequency, try passing 'timedelta64'"
         ),
-        (
-            None,
-            "timedelta64[s]",
-            ValueError,
-            "'timedelta64[s]' is too specific of a frequency, try passing 'timedelta64'",
+    ),
+    (
+        None,
+        "timedelta64[s]",
+        ValueError,
+        re.escape(
+            "'timedelta64[s]' is too specific of a frequency, try passing 'timedelta64'"
         ),
-    ],
+    ),
+]
+
+
+@pytest.mark.parametrize(
+    "include, exclude, exc, exc_match",
+    SELECT_DTYPES_INVALID_PARAMS,
+    ids=[f"include_{a[0]}-exclude_{a[1]}" for a in SELECT_DTYPES_INVALID_PARAMS],
 )
 @sql_count_checker(query_count=0)
 def test_select_dtypes_invalid_args(include, exclude, exc, exc_match):
@@ -178,6 +189,6 @@ def test_select_dtypes_invalid_args(include, exclude, exc, exc_match):
         lambda df: df.select_dtypes(include, exclude),
         expect_exception=True,
         expect_exception_type=exc,
-        expect_exception_match=re.escape(exc_match),
+        expect_exception_match=exc_match,
         assert_exception_equal=True,
     )

tests/integ/test_packaging.py

@@ -22,7 +22,7 @@
 from snowflake.snowpark.functions import call_udf, col, count_distinct, sproc, udf
 from snowflake.snowpark.context import _ANACONDA_SHARED_REPOSITORY
 from snowflake.snowpark.types import DateType, StringType
-from tests.utils import IS_IN_STORED_PROC, TempObjectType, TestFiles, Utils
+from tests.utils import IS_IN_STORED_PROC, TempObjectType, TestFiles, Utils, IS_PY314
 
 pytestmark = pytest.mark.xfail(
     "config.getoption('local_testing_mode', default=False)",
@@ -347,21 +347,23 @@ def test_add_packages_negative(session, caplog):
     assert "InvalidRequirement" in str(ex_info)
 
     session.custom_package_usage_config = {"enabled": True}
-    with patch.object(session, "_is_anaconda_terms_acknowledged", lambda: True):
-        with pytest.raises(RuntimeError, match="Pip failed with return code 1"):
-            session.add_packages("dateutil")
-
-    with patch.object(session, "_is_anaconda_terms_acknowledged", lambda: False):
-        with pytest.raises(RuntimeError, match="Cannot add package dateutil"):
-            session.add_packages("dateutil")
-
-    # Verify multiple errors can be raised at once
-    with patch.object(session, "_is_anaconda_terms_acknowledged", lambda: False):
-        with pytest.raises(
-            RuntimeError,
-            match="Cannot add package dateutil.*Cannot add package functools",
-        ):
-            session.add_packages("dateutil", "functools")
+    # These errors are not raised with Python 3.14 since it uses the pypi repository by default.
+    if not IS_PY314:
+        with patch.object(session, "_is_anaconda_terms_acknowledged", lambda: True):
+            with pytest.raises(RuntimeError, match="Pip failed with return code 1"):
+                session.add_packages("dateutil")
+
+        with patch.object(session, "_is_anaconda_terms_acknowledged", lambda: False):
+            with pytest.raises(RuntimeError, match="Cannot add package dateutil"):
+                session.add_packages("dateutil")
+
+        # Verify multiple errors can be raised at once
+        with patch.object(session, "_is_anaconda_terms_acknowledged", lambda: False):
+            with pytest.raises(
+                RuntimeError,
+                match="Cannot add package dateutil.*Cannot add package functools",
+            ):
+                session.add_packages("dateutil", "functools")
 
     with pytest.raises(ValueError, match="is already added"):
         with caplog.at_level(logging.WARNING):
@@ -464,6 +466,10 @@ def test_add_requirements_twice_should_fail_if_packages_are_different(
     IS_IN_STORED_PROC,
     reason="Subprocess calls are not allowed within stored procedures.",
 )
+@pytest.mark.skipif(
+    IS_PY314,
+    reason="Python 3.14 uses pypi repository",
+)
 def test_add_unsupported_requirements_should_fail_if_custom_packages_upload_enabled_not_switched_on(
     session, resources_path
 ):
@@ -519,6 +525,10 @@ def test_urllib() -> str:
     IS_IN_STORED_PROC,
     reason="Subprocess calls are not allowed within stored procedures.",
 )
+@pytest.mark.skipif(
+    IS_PY314,
+    reason="Python 3.14 uses pypi repository",
+)
 def test_add_unsupported_packages_should_fail_if_custom_packages_upload_enabled_not_switched_on(
     session,
 ):
@@ -991,6 +1001,10 @@ def get_skfuzzy_version() -> str:
     IS_IN_STORED_PROC,
     reason="Subprocess calls are not allowed within stored procedures.",
 )
+@pytest.mark.skipif(
+    IS_PY314,
+    reason="Python 3.14 uses pypi repository",
+)
 def test_add_requirements_unsupported_with_cache_path_negative(
     session, resources_path, temporary_stage
 ):

tests/utils.py

@@ -105,6 +105,8 @@
     "IGNORE_CLIENT_VESRION_IN_STRUCTURED_TYPES_RESPONSE",
 }
 
+IS_PY314 = (sys.version_info.major, sys.version_info.minor) == (3, 14)
+
 
 def current_account(session):
     return session.sql("select CURRENT_ACCOUNT_NAME()").collect()[0][0].upper()