Unskip integ tests mlops (aws#5887)

* test: add spark_py312 marker

* temporarily skip test_to_pipeline_and_execute_with_lake_formation

Author: lucasjia-aws

sagemaker-mlops/tests/integ/feature_store/feature_processor/test_feature_processor_integ.py

@@ -528,10 +528,11 @@ def transform(raw_s3_data_as_df):
         )
 
 
-@pytest.mark.skipif(
-    sys.version_info[:2] not in [(3, 9), (3, 12)],
-    reason=f"SageMaker Spark image only supports Python 3.9 and 3.12, got {sys.version_info[:2]}",
-)
+# @pytest.mark.skipif(
+#     sys.version_info[:2] not in [(3, 9), (3, 12)],
+#     reason=f"SageMaker Spark image only supports Python 3.9 and 3.12, got {sys.version_info[:2]}",
+# )
+@pytest.mark.spark_py312
 @pytest.mark.slow_test
 def test_feature_processor_transform_offline_only_store_ingestion_run_with_remote(
     sagemaker_session,
@@ -669,10 +670,11 @@ def transform(raw_s3_data_as_df):
         )
 
 
-@pytest.mark.skipif(
-    sys.version_info[:2] not in [(3, 9), (3, 12)],
-    reason=f"SageMaker Spark image only supports Python 3.9 and 3.12, got {sys.version_info[:2]}",
-)
+# @pytest.mark.skipif(
+#     sys.version_info[:2] not in [(3, 9), (3, 12)],
+#     reason=f"SageMaker Spark image only supports Python 3.9 and 3.12, got {sys.version_info[:2]}",
+# )
+@pytest.mark.spark_py312
 @pytest.mark.slow_test
 def test_to_pipeline_and_execute(
     sagemaker_session,
@@ -792,10 +794,16 @@ def transform(raw_s3_data_as_df):
         # cleanup_pipeline(pipeline_name="pipeline-name-01", sagemaker_session=sagemaker_session)
 
 
-@pytest.mark.skipif(
-    sys.version_info[:2] not in [(3, 9), (3, 12)],
-    reason=f"SageMaker Spark image only supports Python 3.9 and 3.12, got {sys.version_info[:2]}",
+# @pytest.mark.skipif(
+#     sys.version_info[:2] not in [(3, 9), (3, 12)],
+#     reason=f"SageMaker Spark image only supports Python 3.9 and 3.12, got {sys.version_info[:2]}",
+# )
+@pytest.mark.skip(
+    reason="Lake Formation credential vending (GetTemporaryGlueTableCredentials) requires "
+    "full LF environment setup (resource registration, trust policy, data location grants) "
+    "that is not configured in CI. See quip-amazon.com/S3FEAMMMuKm0 for details."
 )
+@pytest.mark.spark_py312
 @pytest.mark.slow_test
 def test_to_pipeline_and_execute_with_lake_formation(
     sagemaker_session,
@@ -940,10 +948,11 @@ def transform(raw_s3_data_as_df):
             cleanup_feature_group(car_data_fg, sagemaker_session=sagemaker_session)
 
 
-@pytest.mark.skipif(
-    sys.version_info[:2] not in [(3, 9), (3, 12)],
-    reason=f"SageMaker Spark image only supports Python 3.9 and 3.12, got {sys.version_info[:2]}",
-)
+# @pytest.mark.skipif(
+#     sys.version_info[:2] not in [(3, 9), (3, 12)],
+#     reason=f"SageMaker Spark image only supports Python 3.9 and 3.12, got {sys.version_info[:2]}",
+# )
+@pytest.mark.spark_py312
 @pytest.mark.slow_test
 def test_schedule_and_event_trigger(
     sagemaker_session,

sagemaker-mlops/tests/integ/feature_store/feature_processor/test_feature_processor_spark_compat.py

@@ -54,6 +54,7 @@ def test_spark_session_factory_configs_include_dynamic_hadoop():
     assert f"org.apache.hadoop:hadoop-common:{hadoop_version}" in packages
 
 
+@pytest.mark.spark_py312
 @pytest.mark.slow_test
 def test_image_resolver_returns_uri_for_installed_pyspark():
     """Verify the image resolver returns a valid URI for the installed PySpark + Python version."""

sagemaker-mlops/tox.ini

@@ -60,6 +60,7 @@ markers =
     cron
     local_mode
     slow_test
+    spark_py312: mark a test that requires Python 3.12 (Spark image compatibility).
     release
     image_uris_unit_test
     timeout: mark a test as a timeout.