Merge branch 'master' into dacorvo-fix-hf-vllm-image-uri

mollyheamazon · web-flow · commit 1708115a422a · 2026-01-22T12:45:52.000-08:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,14 @@
 # Changelog
+## v3.4.0 (2026-01-22)
+
+### Features
+  - feat: add emr-serverless step for SageMaker Pipelines
+
+### Bug fixes and Other Changes
+  - Add Nova recipe training support in ModelTrainer 
+  - Add Partner-app Auth provider
+  - Add sagemaker dependency for remote function by default V3
+
 
 ## v3.3.1 (2026-01-12)
 ### Bug fixes and Other Changes
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-3.3.1
+3.4.0
diff --git a/pyproject.toml b/pyproject.toml
@@ -33,10 +33,10 @@ classifiers = [
   "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
-  "sagemaker-core>=2.3.1,<3.0.0",
-  "sagemaker-train>=1.3.1,<2.0.0",
-  "sagemaker-serve>=1.3.1,<2.0.0",
-  "sagemaker-mlops>=1.3.1,<2.0.0",
+  "sagemaker-core>=2.4.0,<3.0.0",
+  "sagemaker-train>=1.4.0,<2.0.0",
+  "sagemaker-serve>=1.4.0,<2.0.0",
+  "sagemaker-mlops>=1.4.0,<2.0.0",
 ]
 
 [project.optional-dependencies]
diff --git a/sagemaker-core/CHANGELOG.md b/sagemaker-core/CHANGELOG.md
@@ -1,5 +1,12 @@
 # Changelog
 
+## v2.4.0 (2026-01-22)
+
+### Bug fixes and Other Changes
+
+  - Add Partner-app Auth provider
+  - Add sagemaker dependency for remote function by default V3
+
 ## v2.3.1 (2026-01-12)
 
 ### Bug Fixes
diff --git a/sagemaker-core/VERSION b/sagemaker-core/VERSION
@@ -1,2 +1,2 @@
-2.3.1
+2.4.0
 
diff --git a/sagemaker-core/pyproject.toml b/sagemaker-core/pyproject.toml
@@ -25,7 +25,7 @@ dependencies = [
     "requests>=2.20.0,<3.0.0",
     "attrs>=20.3.0",
     "packaging>=20.0",
-    "protobuf>=3.12,<5.0",
+    "protobuf>=3.12,<7.0.0",
     "pandas>=1.0.0",
     "numpy>=1.9.0",
     # Dependencies migrated from sagemaker_utils
diff --git a/sagemaker-core/tests/unit/image_uris/test_djl.py b/sagemaker-core/tests/unit/image_uris/test_djl.py
@@ -86,6 +86,12 @@ def _test_djl_uris(account, region, version, tag, djl_framework):
 # Known missing framework:version:region combinations that don't exist in ECR
 KNOWN_MISSING_COMBINATIONS = {
     "djl-lmi": {
+        "0.36.0-lmi18.0.0-cu128": {"ap-east-2"},
+        "0.35.0-lmi17.0.0-cu128": {"ap-east-2"},
+        "0.34.0-lmi16.0.0-cu128": {"ap-east-2"},
+        "0.33.0-lmi15.0.0-cu128": {"ap-east-2"},
+        "0.32.0-lmi14.0.0-cu126": {"ap-east-2"},
+        "0.31.0-lmi13.0.0-cu124": {"ap-east-2"},
         "0.30.0-lmi12.0.0-cu124": {"ap-east-2"},
         "0.29.0-lmi11.0.0-cu124": {"ap-east-2"},
         "0.28.0-lmi10.0.0-cu124": {"ap-east-2"},
@@ -205,6 +211,8 @@ def _test_djl_uris(account, region, version, tag, djl_framework):
         },
     },
     "djl-tensorrtllm": {
+        "0.33.0-tensorrtllm0.21.0-cu128": {"ap-east-2"},
+        "0.32.0-tensorrtllm0.12.0-cu125": {"ap-east-2"},
         "0.30.0-tensorrtllm0.12.0-cu125": {"ap-east-2"},
         "0.29.0-tensorrtllm0.11.0-cu124": {"ap-east-2"},
         "0.28.0-tensorrtllm0.9.0-cu122": {"ap-east-2"},
diff --git a/sagemaker-core/tests/unit/test_analytics.py b/sagemaker-core/tests/unit/test_analytics.py
@@ -17,12 +17,8 @@
 import pytest
 from unittest.mock import Mock, patch, MagicMock, mock_open
 from collections import OrderedDict
-
-# Mock pandas before importing analytics
 import sys
 
-sys.modules["pandas"] = MagicMock()
-
 from sagemaker.core.analytics import (
     AnalyticsMetricsBase,
     HyperparameterTuningJobAnalytics,
@@ -33,6 +29,19 @@
 )
 
 
+@pytest.fixture(autouse=True)
+def mock_pandas_for_analytics():
+    """Mock pandas for analytics tests only, with proper cleanup."""
+    original_pandas = sys.modules.get("pandas")
+    sys.modules["pandas"] = MagicMock()
+    yield
+    # Restore original pandas after each test
+    if original_pandas is not None:
+        sys.modules["pandas"] = original_pandas
+    elif "pandas" in sys.modules:
+        del sys.modules["pandas"]
+
+
 class TestAnalyticsMetricsBase:
     """Test AnalyticsMetricsBase abstract class."""
 
diff --git a/sagemaker-mlops/CHANGELOG.md b/sagemaker-mlops/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Changelog
 
+## v1.4.0 (2026-01-22)
+
+* feat: add emr-serverless step for SageMaker Pipelines
+
 ## v1.3.1 (2026-01-12)
 
 * sagemaker-mlops bug fix - Correct source code 'dependencies' parameter to 'requirements'
diff --git a/sagemaker-mlops/VERSION b/sagemaker-mlops/VERSION
@@ -1 +1 @@
-1.3.1
+1.4.0
diff --git a/sagemaker-mlops/pyproject.toml b/sagemaker-mlops/pyproject.toml
@@ -22,9 +22,9 @@ classifiers = [
     "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
-    "sagemaker-core>=2.3.1",
-    "sagemaker-train>=1.3.1",
-    "sagemaker-serve>=1.3.1",
+    "sagemaker-core>=2.4.0",
+    "sagemaker-train>=1.4.0",
+    "sagemaker-serve>=1.4.0",
     "boto3>=1.42.2,<2.0",
     "botocore>=1.42.2,<2.0",
 ]
diff --git a/sagemaker-serve/CHANGELOG.md b/sagemaker-serve/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Changelog
 
+## v1.4.0 (2026-01-22)
+
+* Update Module dependencies
+
 ## v1.3.1 (2026-01-12)
 
 * Update Module dependencies
diff --git a/sagemaker-serve/VERSION b/sagemaker-serve/VERSION
@@ -1 +1 @@
-1.3.1
+1.4.0
diff --git a/sagemaker-serve/pyproject.toml b/sagemaker-serve/pyproject.toml
@@ -22,8 +22,8 @@ classifiers = [
     "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
-    "sagemaker-core>=2.3.1",
-    "sagemaker-train>=1.3.1",
+    "sagemaker-core>=2.4.0",
+    "sagemaker-train>=1.4.0",
     "boto3>=1.42.2,<2.0",
     "botocore>=1.35.75,<2.0",
     "deepdiff",
diff --git a/sagemaker-serve/src/sagemaker/serve/model_builder_utils.py b/sagemaker-serve/src/sagemaker/serve/model_builder_utils.py
@@ -1391,12 +1391,14 @@ def _get_artifact_path(self, mlflow_model_path: str) -> str:
             if "@" in mlflow_model_path:
                 _, model_name_and_alias, artifact_uri = mlflow_model_path.split("/", 2)
                 model_name, model_alias = model_name_and_alias.split("@")
-                model_metadata = mlflow_client.get_model_version_by_alias(model_name, model_alias)
+                model_version_info = mlflow_client.get_model_version_by_alias(model_name, model_alias)
+                source = mlflow_client.get_model_version_download_uri(
+                    model_name, model_version_info.version
+                )
             else:
                 _, model_name, model_version, artifact_uri = mlflow_model_path.split("/", 3)
-                model_metadata = mlflow_client.get_model_version(model_name, model_version)
+                source = mlflow_client.get_model_version_download_uri(model_name, model_version)
 
-            source = model_metadata.source
             if not source.endswith("/"):
                 source += "/"
             return source + artifact_uri
diff --git a/sagemaker-serve/src/sagemaker/serve/utils/conda_in_process.yml b/sagemaker-serve/src/sagemaker/serve/utils/conda_in_process.yml
@@ -13,7 +13,7 @@ dependencies:
     - cloudpickle==2.2.1
     - google-pasta
     - numpy>=2.0.0,<3.0
-    - protobuf>=3.12,<5.0
+    - protobuf>=3.12,<7.0
     - smdebug_rulesconfig==1.0.1
     - importlib-metadata>=1.4.0,<7.0
     - packaging>=20.0
diff --git a/sagemaker-serve/tests/integ/test_huggingface_integration.py b/sagemaker-serve/tests/integ/test_huggingface_integration.py
@@ -25,7 +25,7 @@
 logger = logging.getLogger(__name__)
 
 # Configuration - easily customizable
-MODEL_ID = "t5-small"  # Small text generation model
+MODEL_ID = "gpt2"  # Small causal language model compatible with DJL 0.36.0 vLLM
 MODEL_NAME_PREFIX = "hf-test-model"
 ENDPOINT_NAME_PREFIX = "hf-test-endpoint"
 
diff --git a/sagemaker-train/CHANGELOG.md b/sagemaker-train/CHANGELOG.md
@@ -1,5 +1,11 @@
 # Changelog
 
+## v1.4.0 (2026-01-22)
+
+### Bug fixes and Other Changes
+
+* Add Nova training support in Model Trainer
+
 ## v1.3.1 (2026-01-12)
 
 ### Bug fixes and Other Changes
diff --git a/sagemaker-train/VERSION b/sagemaker-train/VERSION
@@ -1 +1 @@
-1.3.1
+1.4.0
diff --git a/sagemaker-train/pyproject.toml b/sagemaker-train/pyproject.toml
@@ -33,7 +33,7 @@ classifiers = [
   "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
-    "sagemaker-core>=2.3.1",
+    "sagemaker-core>=2.4.0",
     "graphene>=3,<4",
     "typing_extensions>=4.9.0",
     "tblib>=1.7.0",
diff --git a/sagemaker-train/src/sagemaker/train/constants.py b/sagemaker-train/src/sagemaker/train/constants.py
@@ -62,3 +62,7 @@
     "mistral.mistral-large-2402-v1:0": ["us-west-2", "us-east-1", "eu-west-1"],
     "amazon.nova-pro-v1:0": ["us-east-1"]
 }
+
+SM_RECIPE = "recipe"
+SM_RECIPE_YAML = "recipe.yaml"
+SM_RECIPE_CONTAINER_PATH = f"/opt/ml/input/data/recipe/{SM_RECIPE_YAML}"
diff --git a/sagemaker-train/src/sagemaker/train/model_trainer.py b/sagemaker-train/src/sagemaker/train/model_trainer.py
@@ -83,6 +83,9 @@
     SM_CODE_CONTAINER_PATH,
     SM_DRIVERS,
     SM_DRIVERS_LOCAL_PATH,
+    SM_RECIPE,
+    SM_RECIPE_YAML,
+    SM_RECIPE_CONTAINER_PATH,
     TRAIN_SCRIPT,
     DEFAULT_CONTAINER_ENTRYPOINT,
     DEFAULT_CONTAINER_ARGUMENTS,
@@ -100,7 +103,13 @@
 from sagemaker.core.telemetry.telemetry_logging import _telemetry_emitter
 from sagemaker.core.telemetry.constants import Feature
 from sagemaker.train import logger
-from sagemaker.train.sm_recipes.utils import _get_args_from_recipe, _determine_device_type
+from sagemaker.train.sm_recipes.utils import (
+    _get_args_from_recipe,
+    _determine_device_type,
+    _is_nova_recipe,
+    _is_llmft_recipe,
+    _load_base_recipe,
+)
 
 from sagemaker.core.jumpstart.configs import JumpStartConfig
 from sagemaker.core.jumpstart.document import get_hub_content_and_document
@@ -249,6 +258,8 @@ class ModelTrainer(BaseModel):
     _remote_debug_config: Optional[RemoteDebugConfig] = PrivateAttr(default=None)
     _metric_definitions: Optional[List[MetricDefinition]] = PrivateAttr(default=None)
 
+    _is_nova_recipe: Optional[bool] = PrivateAttr(default=None)
+    _is_llmft_recipe: Optional[bool] = PrivateAttr(default=None)
     # Private Attributes for Recipes
     _temp_recipe_train_dir: Optional[TemporaryDirectory] = PrivateAttr(default=None)
 
@@ -573,6 +584,23 @@ def _create_training_job_args(
 
             final_input_data_config = list(existing_channels.values()) + new_channels
 
+        if self._is_nova_recipe or self._is_llmft_recipe:
+            for input_data in final_input_data_config:
+                if input_data.channel_name == SM_RECIPE:
+                    raise ValueError(
+                        "Cannot use reserved channel name 'recipe' as an input channel name "
+                        " for Nova or LLMFT Recipe"
+                    )
+            recipe_file_path = os.path.join(self._temp_recipe_train_dir.name, SM_RECIPE_YAML)
+            recipe_channel = self.create_input_data_channel(
+                channel_name=SM_RECIPE,
+                data_source=recipe_file_path,
+                key_prefix=input_data_key_prefix,
+            )
+            final_input_data_config.append(recipe_channel)
+            if self._is_nova_recipe or self._is_llmft_recipe:
+                self.hyperparameters.update({"sagemaker_recipe_local_path": SM_RECIPE_CONTAINER_PATH})
+
         if final_input_data_config:
             final_input_data_config = self._get_input_data_config(
                 final_input_data_config, input_data_key_prefix
@@ -1039,6 +1067,7 @@ def from_recipe(
         checkpoint_config: Optional[shapes.CheckpointConfig] = None,
         training_input_mode: Optional[str] = "File",
         environment: Optional[Dict[str, str]] = None,
+        hyperparameters: Optional[Union[Dict[str, Any], str]] = {},
         tags: Optional[List[Tag]] = None,
         sagemaker_session: Optional[Session] = None,
         role: Optional[str] = None,
@@ -1136,12 +1165,20 @@ def from_recipe(
         if compute.instance_type is None:
             raise ValueError("Must set ``instance_type`` in Compute when using training recipes.")
         device_type = _determine_device_type(compute.instance_type)
-        if device_type == "cpu":
+        recipe = _load_base_recipe(
+            training_recipe=training_recipe, recipe_overrides=recipe_overrides
+        )
+        is_nova = _is_nova_recipe(recipe=recipe)
+        is_llmft = _is_llmft_recipe(recipe=recipe)
+        if device_type == "cpu" and not (is_nova or is_llmft):
             raise ValueError(
                 "Training recipes are not supported for CPU instances. "
                 "Please provide a GPU or Tranium instance type."
             )
 
+        if training_image is None and (is_nova or is_llmft):
+            raise ValueError("training_image must be provided when using recipe for Nova or LLMFT")
+
         if training_image_config and training_image is None:
             raise ValueError("training_image must be provided when using training_image_config.")
 
@@ -1154,16 +1191,29 @@ def from_recipe(
         # - distributed
         # - compute
         # - hyperparameters
-        model_trainer_args, recipe_train_dir = _get_args_from_recipe(
-            training_recipe=training_recipe,
+        model_trainer_args, tmp_dir = _get_args_from_recipe(
+            training_recipe=recipe,
             recipe_overrides=recipe_overrides,
             requirements=requirements,
             compute=compute,
             region_name=sagemaker_session.boto_region_name,
+            role=role,
         )
         if training_image is not None:
             model_trainer_args["training_image"] = training_image
 
+        if hyperparameters and not is_nova:
+            logger.warning(
+                "Hyperparameters are not supported for general and LLMFT training recipes. "
+                + "Ignoring hyperparameters input."
+            )
+        if is_nova:
+            if hyperparameters and isinstance(hyperparameters, str):
+                hyperparameters = cls._validate_and_load_hyperparameters_file(hyperparameters)
+                model_trainer_args["hyperparameters"].update(hyperparameters)
+            elif hyperparameters and isinstance(hyperparameters, dict):
+                model_trainer_args["hyperparameters"].update(hyperparameters)
+
         model_trainer = cls(
             sagemaker_session=sagemaker_session,
             role=role,
@@ -1180,7 +1230,9 @@ def from_recipe(
             **model_trainer_args,
         )
 
-        model_trainer._temp_recipe_train_dir = recipe_train_dir
+        model_trainer._is_nova_recipe = is_nova
+        model_trainer._is_llmft_recipe = is_llmft
+        model_trainer._temp_recipe_train_dir = tmp_dir
         return model_trainer
 
     @classmethod
diff --git a/sagemaker-train/src/sagemaker/train/sm_recipes/utils.py b/sagemaker-train/src/sagemaker/train/sm_recipes/utils.py
diff --git a/sagemaker-train/tests/unit/train/sm_recipes/test_utils.py b/sagemaker-train/tests/unit/train/sm_recipes/test_utils.py
diff --git a/sagemaker-train/tests/unit/train/test_model_trainer.py b/sagemaker-train/tests/unit/train/test_model_trainer.py

Original file line number	Diff line number	Diff line change
`@@ -33,10 +33,10 @@ classifiers = [`
`33`	`33`	`"Programming Language :: Python :: 3.12",`
`34`	`34`	`]`
`35`	`35`	`dependencies = [`
`36`		`- "sagemaker-core>=2.3.1,<3.0.0",`
`37`		`- "sagemaker-train>=1.3.1,<2.0.0",`
`38`		`- "sagemaker-serve>=1.3.1,<2.0.0",`
`39`		`- "sagemaker-mlops>=1.3.1,<2.0.0",`
	`36`	`+ "sagemaker-core>=2.4.0,<3.0.0",`
	`37`	`+ "sagemaker-train>=1.4.0,<2.0.0",`
	`38`	`+ "sagemaker-serve>=1.4.0,<2.0.0",`
	`39`	`+ "sagemaker-mlops>=1.4.0,<2.0.0",`
`40`	`40`	`]`
`41`	`41`
`42`	`42`	`[project.optional-dependencies]`