fix spark session bug

Author: bhhalim

sagemaker-mlops/src/sagemaker/mlops/feature_store/feature_processor/_spark_factory.py

@@ -180,7 +180,7 @@ def get_spark_session_with_iceberg_config(self, warehouse_s3_uri, catalog) -> Sp
             SparkSession: A SparkSession ready to support reading and writing data from an Iceberg
                 Table.
         """
-        conf = self.spark_session._jvm.SparkSession().conf()
+        conf = self.spark_session.conf
 
         for cfg in self._get_iceberg_configs(warehouse_s3_uri, catalog):
             conf.set(cfg[0], cfg[1])

sagemaker-mlops/tests/unit/sagemaker/mlops/feature_store/feature_processor/test_spark_session_factory.py

@@ -117,19 +117,20 @@ def test_spark_session_factory_with_iceberg_config(mock_spark_context):
     spark_session_factory = SparkSessionFactory(mock_env_helper)
 
     spark_session = spark_session_factory.spark_session
+    mock_conf = Mock()
 
-    spark_session_with_iceberg_config = spark_session_factory.get_spark_session_with_iceberg_config(
-        "warehouse", "catalog"
-    )
+    with patch.object(type(spark_session), "conf", new_callable=lambda: property(lambda self: mock_conf)):
+        spark_session_with_iceberg_config = spark_session_factory.get_spark_session_with_iceberg_config(
+            "warehouse", "catalog"
+        )
 
-    assert spark_session is spark_session_with_iceberg_config
-    mock_spark_conf = spark_session._jvm.SparkSession().conf()
-    expected_calls = [
-        call.set(cfg[0], cfg[1])
-        for cfg in spark_session_factory._get_iceberg_configs("warehouse", "catalog")
-    ]
+        assert spark_session is spark_session_with_iceberg_config
+        expected_calls = [
+            call.set(cfg[0], cfg[1])
+            for cfg in spark_session_factory._get_iceberg_configs("warehouse", "catalog")
+        ]
 
-    mock_spark_conf.assert_has_calls(expected_calls, any_order=False)
+        mock_conf.assert_has_calls(expected_calls, any_order=False)
 
 
 @patch("pyspark.context.SparkContext.getOrCreate")