fix: [Bug]: PySDK V3: pydantic-core (2.42.0) incompatibility with sagemaker.ai_regist (5652)

Author: aviruthen

sagemaker-core/pyproject.toml

@@ -13,7 +13,8 @@ readme = "README.rst"
 dependencies = [
   # Add your dependencies here (Include lower and upper bounds as applicable)
     "boto3>=1.42.2,<2.0.0",
-    "pydantic>=2.0.0,<3.0.0",
+    "pydantic>=2.10.0,<3.0.0",
+    "pydantic-core>=2.27.0,<3.0.0",
     "PyYAML>=6.0, <7.0",
     "jsonschema<5.0.0",
     "platformdirs>=4.0.0, <5.0.0",

sagemaker-core/src/sagemaker/core/__init__.py

@@ -1,3 +1,14 @@
+# Early pydantic compatibility check - must happen before any pydantic imports
+try:
+    from sagemaker.core._pydantic_compat import check_pydantic_compatibility
+    check_pydantic_compatibility()
+except ImportError as e:
+    if "pydantic" in str(e).lower() and ("incompatible" in str(e).lower() or "mismatch" in str(e).lower()):
+        raise
+    # If it's a different ImportError (e.g., pydantic not installed yet), let it pass
+    # and fail later with a more standard error
+    pass
+
 from sagemaker.core.utils.utils import enable_textual_rich_console_and_traceback
 
 

sagemaker-core/src/sagemaker/core/_pydantic_compat.py

@@ -0,0 +1,80 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Pydantic compatibility check for sagemaker-core.
+
+This module provides an early check for pydantic/pydantic-core version
+compatibility to give users a clear error message with fix instructions
+instead of a cryptic SystemError.
+"""
+
+
+def check_pydantic_compatibility():
+    """Check that pydantic and pydantic-core versions are compatible.
+
+    Raises:
+        ImportError: If pydantic and pydantic-core versions are incompatible,
+            with instructions on how to fix the issue.
+    """
+    try:
+        import pydantic  # noqa: F401
+    except SystemError as e:
+        error_message = str(e)
+        raise ImportError(
+            f"Pydantic version incompatibility detected: {error_message}\n\n"
+            "This typically happens when pydantic-core is upgraded independently "
+            "of pydantic, causing a version mismatch.\n\n"
+            "To fix this, run:\n"
+            "    pip install pydantic pydantic-core --force-reinstall\n\n"
+            "This will ensure both packages are installed at compatible versions."
+        ) from e
+
+    try:
+        import pydantic_core  # noqa: F401
+    except ImportError:
+        # pydantic_core not installed separately is fine;
+        # pydantic manages it as a dependency
+        return
+
+    # Additional version check: pydantic declares the exact pydantic-core
+    # version it requires. Verify they match.
+    try:
+        pydantic_version = pydantic.VERSION
+        pydantic_core_version = pydantic_core.VERSION
+
+        # pydantic >= 2.x stores the required core version
+        expected_core_version = getattr(pydantic, '__pydantic_core_version__', None)
+        if expected_core_version is None:
+            # Try alternative attribute name used in some pydantic versions
+            expected_core_version = getattr(
+                pydantic, '_internal', None
+            ) and getattr(
+                getattr(pydantic, '_internal', None),
+                '_generate_schema',
+                None,
+            )
+            # If we can't determine the expected version, skip the check
+            return
+
+        if pydantic_core_version != expected_core_version:
+            raise ImportError(
+                f"Pydantic/pydantic-core version mismatch detected: "
+                f"pydantic {pydantic_version} requires pydantic-core=={expected_core_version}, "
+                f"but pydantic-core {pydantic_core_version} is installed.\n\n"
+                "To fix this, run:\n"
+                "    pip install pydantic pydantic-core --force-reinstall\n\n"
+                "This will ensure both packages are installed at compatible versions."
+            )
+    except (AttributeError, TypeError):
+        # If we can't determine versions, skip the check
+        # The SystemError catch above will handle the most common case
+        pass

sagemaker-core/tests/unit/test_pydantic_compat.py

@@ -0,0 +1,76 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Tests for pydantic compatibility check."""
+
+import sys
+from unittest import mock
+
+import pytest
+
+
+def test_check_pydantic_compatibility_passes_with_matching_versions():
+    """Verify the check function does not raise when pydantic and pydantic-core are compatible."""
+    from sagemaker.core._pydantic_compat import check_pydantic_compatibility
+
+    # Should not raise any exception with the currently installed versions
+    check_pydantic_compatibility()
+
+
+def test_check_pydantic_compatibility_raises_on_system_error():
+    """Mock pydantic import to raise SystemError and verify a clear ImportError is raised."""
+    from sagemaker.core._pydantic_compat import check_pydantic_compatibility
+
+    error_msg = (
+        "The installed pydantic-core version (2.42.0) is incompatible "
+        "with the current pydantic version, which requires 2.41.5."
+    )
+
+    with mock.patch.dict(sys.modules, {"pydantic": None}):
+        original_import = __builtins__.__import__ if hasattr(__builtins__, '__import__') else __import__
+
+        def mock_import(name, *args, **kwargs):
+            if name == "pydantic":
+                raise SystemError(error_msg)
+            return original_import(name, *args, **kwargs)
+
+        with mock.patch("builtins.__import__", side_effect=mock_import):
+            with pytest.raises(ImportError) as exc_info:
+                check_pydantic_compatibility()
+
+            assert "incompatibility detected" in str(exc_info.value).lower() or \
+                   "incompatible" in str(exc_info.value).lower()
+
+
+def test_pydantic_import_error_message_contains_instructions():
+    """Verify the error message includes pip install instructions."""
+    from sagemaker.core._pydantic_compat import check_pydantic_compatibility
+
+    error_msg = (
+        "The installed pydantic-core version (2.42.0) is incompatible "
+        "with the current pydantic version, which requires 2.41.5."
+    )
+
+    with mock.patch.dict(sys.modules, {"pydantic": None}):
+        original_import = __builtins__.__import__ if hasattr(__builtins__, '__import__') else __import__
+
+        def mock_import(name, *args, **kwargs):
+            if name == "pydantic":
+                raise SystemError(error_msg)
+            return original_import(name, *args, **kwargs)
+
+        with mock.patch("builtins.__import__", side_effect=mock_import):
+            with pytest.raises(ImportError) as exc_info:
+                check_pydantic_compatibility()
+
+            error_str = str(exc_info.value)
+            assert "pip install pydantic pydantic-core --force-reinstall" in error_str