fix: update error handling for SparkSession retrieval in Databricks adapter

### Description

This commit updates the error messages in the `DatabricksSessionHandle` and `SessionPythonJobHelper` classes to provide clearer guidance when a SparkSession is not found. It emphasizes the compatibility requirements for session mode execution in Databricks, ensuring users are informed about the necessary environments and configurations.

Author: alexeyegorov

dbt/adapters/databricks/__version__.py

@@ -1 +1 @@
-version = "1.10.15-2"
+version = "1.10.15-3"

dbt/adapters/databricks/python_models/python_submissions.py

@@ -846,33 +846,30 @@ def __init__(self, parsed_model: dict,
                     f"[Session Python] Could not get SparkSession from __main__.spark: {e}"
                 )
 
-        # Method 4: Get the shared SparkContext on Databricks and create SparkSession from it
-        # On Databricks, we must use the shared SparkContext - we cannot create a new one
+        # If no existing SparkSession found, provide a clear error message
         if spark is None:
             databricks_runtime = os.getenv("DATABRICKS_RUNTIME_VERSION")
             if databricks_runtime:
-                logger.debug(
-                    f"[Session Python] No existing SparkSession found. "
-                    f"Getting shared SparkContext on Databricks runtime {databricks_runtime}"
-                )
-                try:
-                    # On Databricks, we MUST use SparkContext.getOrCreate() to get the shared context
-                    # Creating a new SparkContext is not allowed
-                    sc = SparkContext.getOrCreate()
-                    spark = SparkSession(sc)
-                    logger.debug(
-                        "[Session Python] Created SparkSession from shared SparkContext"
-                    )
-                except Exception as create_error:
-                    raise DbtRuntimeError(
-                        f"[Session Python] Failed to get shared SparkContext on Databricks: "
-                        f"{create_error}. Databricks runtime version: {databricks_runtime}"
-                    ) from create_error
+                raise DbtRuntimeError(
+                    "[Session Python] Could not find an existing SparkSession. "
+                    "This typically happens when using the native 'dbt task' in Databricks Jobs, "
+                    "which does not provide a SparkSession context.\n\n"
+                    "Session mode is only compatible with:\n"
+                    "  - Databricks Notebooks (where 'spark' is pre-initialized)\n"
+                    "  - Python tasks that initialize SparkSession before running dbt\n"
+                    "  - Environments where SparkSession is already available\n\n"
+                    "For the native dbt task, use DBSQL mode instead (the default):\n"
+                    "  - Set 'method: dbsql' in your profile (or omit 'method' entirely)\n"
+                    "  - Configure 'host' and 'http_path' to connect to a SQL warehouse or cluster\n\n"
+                    f"Databricks runtime version: {databricks_runtime}")
             else:
                 raise DbtRuntimeError(
-                    "[Session Python] Session mode requires a Databricks cluster environment. "
+                    "[Session Python] Session mode requires a Databricks cluster environment "
+                    "with an active SparkSession. "
                     "DATABRICKS_RUNTIME_VERSION environment variable not found. "
-                    "Ensure you are running on a Databricks cluster.")
+                    "Ensure you are running on a Databricks cluster in a context where "
+                    "SparkSession is available (e.g., Notebook or Python task with Spark initialized)."
+                )
 
         self._spark = spark
         logger.debug(

dbt/adapters/databricks/session.py

@@ -212,32 +212,29 @@ def create(
                 logger.debug(
                     f"Could not get SparkSession from __main__.spark: {e}")
 
-        # Method 4: Get the shared SparkContext on Databricks and create SparkSession from it
-        # On Databricks, we must use the shared SparkContext - we cannot create a new one
+        # If no existing SparkSession found, provide a clear error message
         if spark is None:
             databricks_runtime = os.getenv("DATABRICKS_RUNTIME_VERSION")
             if databricks_runtime:
-                logger.debug(
-                    f"No existing SparkSession found. "
-                    f"Getting shared SparkContext on Databricks runtime {databricks_runtime}"
-                )
-                try:
-                    # On Databricks, we MUST use SparkContext.getOrCreate() to get the shared context
-                    # Creating a new SparkContext is not allowed
-                    sc = SparkContext.getOrCreate()
-                    spark = SparkSession(sc)
-                    logger.debug(
-                        "Created SparkSession from shared SparkContext")
-                except Exception as create_error:
-                    raise DbtRuntimeError(
-                        f"Failed to get shared SparkContext on Databricks: {create_error}. "
-                        f"Databricks runtime version: {databricks_runtime}"
-                    ) from create_error
+                raise DbtRuntimeError(
+                    "Session mode could not find an existing SparkSession. "
+                    "This typically happens when using the native 'dbt task' in Databricks Jobs, "
+                    "which does not provide a SparkSession context.\n\n"
+                    "Session mode is only compatible with:\n"
+                    "  - Databricks Notebooks (where 'spark' is pre-initialized)\n"
+                    "  - Python tasks that initialize SparkSession before running dbt\n"
+                    "  - Environments where SparkSession is already available\n\n"
+                    "For the native dbt task, use DBSQL mode instead (the default):\n"
+                    "  - Set 'method: dbsql' in your profile (or omit 'method' entirely)\n"
+                    "  - Configure 'host' and 'http_path' to connect to a SQL warehouse or cluster\n\n"
+                    f"Databricks runtime version: {databricks_runtime}")
             else:
                 raise DbtRuntimeError(
-                    "Session mode requires a Databricks cluster environment. "
+                    "Session mode requires a Databricks cluster environment with an active SparkSession. "
                     "DATABRICKS_RUNTIME_VERSION environment variable not found. "
-                    "Ensure you are running on a Databricks cluster.")
+                    "Ensure you are running on a Databricks cluster in a context where "
+                    "SparkSession is available (e.g., Notebook or Python task with Spark initialized)."
+                )
 
         # Set catalog if provided
         if catalog: