fix: update SparkSession retrieval method in Databricks adapter

### Description

This commit modifies the `DatabricksSessionHandle` and `SessionPythonJobHelper` classes to ensure the retrieval of the SparkSession uses the shared SparkContext instead of creating a new one. This change enhances compatibility with Databricks environments and improves error handling during SparkSession initialization.

Author: alexeyegorov

dbt/adapters/databricks/__version__.py

@@ -1 +1 @@
-version = "1.10.15d"
+version = "1.10.15-2"

dbt/adapters/databricks/python_models/python_submissions.py

@@ -846,25 +846,26 @@ def __init__(self, parsed_model: dict,
                     f"[Session Python] Could not get SparkSession from __main__.spark: {e}"
                 )
 
-        # Method 4: Create a new SparkSession on Databricks
-        # If no existing session found, create one using local[*] master
-        # This works on Databricks because the Spark runtime is already configured
+        # Method 4: Get the shared SparkContext on Databricks and create SparkSession from it
+        # On Databricks, we must use the shared SparkContext - we cannot create a new one
         if spark is None:
             databricks_runtime = os.getenv("DATABRICKS_RUNTIME_VERSION")
             if databricks_runtime:
                 logger.debug(
                     f"[Session Python] No existing SparkSession found. "
-                    f"Creating new SparkSession on Databricks runtime {databricks_runtime}"
+                    f"Getting shared SparkContext on Databricks runtime {databricks_runtime}"
                 )
                 try:
-                    spark = (SparkSession.builder.master("local[*]").appName(
-                        "dbt-databricks-session-python").getOrCreate())
+                    # On Databricks, we MUST use SparkContext.getOrCreate() to get the shared context
+                    # Creating a new SparkContext is not allowed
+                    sc = SparkContext.getOrCreate()
+                    spark = SparkSession(sc)
                     logger.debug(
-                        "[Session Python] Created new SparkSession for session mode"
+                        "[Session Python] Created SparkSession from shared SparkContext"
                     )
                 except Exception as create_error:
                     raise DbtRuntimeError(
-                        f"[Session Python] Failed to create SparkSession on Databricks: "
+                        f"[Session Python] Failed to get shared SparkContext on Databricks: "
                         f"{create_error}. Databricks runtime version: {databricks_runtime}"
                     ) from create_error
             else:

dbt/adapters/databricks/session.py

@@ -212,23 +212,25 @@ def create(
                 logger.debug(
                     f"Could not get SparkSession from __main__.spark: {e}")
 
-        # Method 4: Create a new SparkSession on Databricks
-        # If no existing session found, create one using local[*] master
-        # This works on Databricks because the Spark runtime is already configured
+        # Method 4: Get the shared SparkContext on Databricks and create SparkSession from it
+        # On Databricks, we must use the shared SparkContext - we cannot create a new one
         if spark is None:
             databricks_runtime = os.getenv("DATABRICKS_RUNTIME_VERSION")
             if databricks_runtime:
                 logger.debug(
                     f"No existing SparkSession found. "
-                    f"Creating new SparkSession on Databricks runtime {databricks_runtime}"
+                    f"Getting shared SparkContext on Databricks runtime {databricks_runtime}"
                 )
                 try:
-                    spark = (SparkSession.builder.master("local[*]").appName(
-                        "dbt-databricks-session").getOrCreate())
-                    logger.debug("Created new SparkSession for session mode")
+                    # On Databricks, we MUST use SparkContext.getOrCreate() to get the shared context
+                    # Creating a new SparkContext is not allowed
+                    sc = SparkContext.getOrCreate()
+                    spark = SparkSession(sc)
+                    logger.debug(
+                        "Created SparkSession from shared SparkContext")
                 except Exception as create_error:
                     raise DbtRuntimeError(
-                        f"Failed to create SparkSession on Databricks: {create_error}. "
+                        f"Failed to get shared SparkContext on Databricks: {create_error}. "
                         f"Databricks runtime version: {databricks_runtime}"
                     ) from create_error
             else: