Add kernel integration test lane + use_kernel auth support

sd-db · sd-db · commit ef3c48264073 · 2026-07-06T18:42:48.000+05:30
Adds a weekly SQL-warehouse integration workflow that runs the functional
suite through the databricks-sql-connector Rust kernel backend
(use_kernel=True), and the adapter change needed for connections to
authenticate on that path.

The kernel routes over SEA and owns the OAuth token lifecycle itself, so
its auth bridge rejects the connector credentials_provider dbt passes on
the Thrift path. prepare_connection_arguments now forwards the raw
credential the bridge understands (PAT, Databricks OAuth M2M, or OAuth
U2M) when use_kernel is set, and raises a clear error for native Azure
service-principal auth (which the kernel cannot serve) rather than
misrouting it. SEA is warehouse-only, so the workflow runs only the
databricks_uc_sql_endpoint profile and tests that pin a model to a
cluster are marked @pytest.mark.skip_kernel.
diff --git a/.github/workflows/integration-kernel.yml b/.github/workflows/integration-kernel.yml
@@ -0,0 +1,182 @@
+# Kernel Integration Tests for dbt-databricks.
+#
+# Runs the functional suite against the SQL-warehouse profile with the
+# databricks-sql-connector Rust kernel backend (use_kernel=True) instead of
+# the default Thrift backend, on a weekly Saturday schedule. Like
+# integration-spog.yml this is a scheduled smoke, not a PR gate — PR-targeting
+# and status-reporting are omitted.
+#
+# Why warehouse-only: the kernel routes over the SEA (Statement Execution API)
+# HTTP transport, which is warehouse-only — the connector rejects an
+# all-purpose-cluster http_path ("SEA only works for warehouses"). So the two
+# cluster profiles from the main matrix cannot run on the kernel and are not
+# included here.
+#
+# Auth: the kernel owns the OAuth token lifecycle itself and its auth bridge
+# rejects the connector credentials_provider dbt passes on the Thrift path.
+# When DBT_DATABRICKS_USE_KERNEL=1, tests/profiles.py sets use_kernel=True and
+# the adapter forwards the raw OAuth M2M client id/secret (or a PAT) instead —
+# see SqlUtils.prepare_connection_arguments.
+name: Kernel Integration Tests
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "17 9 * * 6"  # Weekly: Saturday 09:17 UTC (14:47 IST). Clear of the
+                          # daily 21:30 UTC nightly and the Sunday SPOG slot,
+                          # which share the peco warehouse.
+
+permissions:
+  id-token: write
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  prepare-shards:
+    runs-on:
+      group: databricks-protected-runner-group
+      labels: linux-ubuntu-latest
+    env:
+      UV_FROZEN: "1"
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
+
+      - name: Setup JFrog PyPI Proxy
+        uses: ./.github/actions/setup-jfrog-pypi
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
+        with:
+          python-version: "3.10"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a  # v4
+        with:
+          cache-local-path: ~/.cache/uv
+
+      - name: Install Hatch
+        uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc  # install
+
+      - name: Collect tests and assign to shards
+        run: |
+          set -euo pipefail
+          mkdir -p shard-assignments
+          hatch run pytest --collect-only -q --profile databricks_uc_sql_endpoint tests/functional 2>&1 \
+            | grep "::" \
+            > "shard-assignments/databricks_uc_sql_endpoint-collected.txt"
+          python3 scripts/shard_assign.py \
+            --profile databricks_uc_sql_endpoint \
+            --num-shards 3 \
+            --input "shard-assignments/databricks_uc_sql_endpoint-collected.txt" \
+            --output-dir shard-assignments \
+            --algo lpt_historical_time \
+            --timings .github/test_timings.json
+
+      - name: Upload shard assignments
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4
+        with:
+          name: shard-assignments-kernel
+          path: shard-assignments/
+          retention-days: 5
+
+  uc-sql-endpoint:
+    needs: prepare-shards
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [0, 1, 2]
+    runs-on:
+      group: databricks-protected-runner-group
+      labels: linux-ubuntu-latest
+    environment: azure-prod
+    env:
+      DBT_DATABRICKS_HOST_NAME: ${{ secrets.DATABRICKS_HOST }}
+      DBT_DATABRICKS_CLIENT_ID: ${{ secrets.TEST_PECO_SP_ID }}
+      DBT_DATABRICKS_CLIENT_SECRET: ${{ secrets.TEST_PECO_SP_SECRET }}
+      DBT_DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }}
+      DBT_DATABRICKS_UC_INITIAL_CATALOG: peco
+      DBT_DATABRICKS_LOCATION_ROOT: ${{ secrets.TEST_PECO_EXTERNAL_LOCATION }}test
+      # Wire the UC cluster id so build_cluster_http_path.py can populate
+      # DBT_DATABRICKS_UC_CLUSTER_HTTP_PATH. The kernel-incompatible tests that
+      # pin a model to that cluster path are @pytest.mark.skip_kernel and skip
+      # here; this keeps the env var resolvable so their fixtures still parse.
+      TEST_PECO_UC_CLUSTER_ID: ${{ secrets.TEST_PECO_UC_CLUSTER_ID }}
+      TEST_PECO_SPOG_HOST: ${{ secrets.TEST_PECO_SPOG_HOST }}
+      TEST_PECO_SPOG_WORKSPACE_ID: ${{ secrets.TEST_PECO_SPOG_WORKSPACE_ID }}
+      # Route the whole suite through the connector's Rust kernel (SEA) backend.
+      DBT_DATABRICKS_USE_KERNEL: "1"
+      UV_FROZEN: "1"
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
+
+      - name: Setup JFrog PyPI Proxy
+        uses: ./.github/actions/setup-jfrog-pypi
+
+      - name: Set up python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
+        with:
+          python-version: "3.10"
+
+      - name: Get http path from environment
+        run: python .github/workflows/build_cluster_http_path.py
+        shell: sh
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a  # v4
+        with:
+          cache-local-path: ~/.cache/uv
+
+      - name: Install Hatch
+        uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc  # install
+
+      - name: Install the connector kernel extra
+        # The kernel backend ships in the optional databricks-sql-connector
+        # [kernel] extra (a PyO3 wheel), which the frozen lock does not include.
+        # Add it into the hatch env so use_kernel=True can import the kernel.
+        run: |
+          set -euo pipefail
+          HATCH_PY="$(hatch run python -c 'import sys; print(sys.executable)')"
+          hatch run python -c "import databricks.sql as s; print('connector version:', s.__version__)"
+          uv pip install --python "$HATCH_PY" "databricks-sql-connector[kernel]"
+          hatch run python -c "import databricks_sql_kernel; print('kernel extra installed')"
+
+      - name: Download shard assignments
+        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e  # v4.1.7
+        with:
+          name: shard-assignments-kernel
+          path: shard-assignments/
+
+      - name: Resolve test list for this shard
+        run: |
+          set -euo pipefail
+          SHARD_FILE="shard-assignments/databricks_uc_sql_endpoint-shard-${{ matrix.shard }}.txt"
+          if [ ! -s "$SHARD_FILE" ]; then
+            echo "::error::Shard file missing or empty: $SHARD_FILE"
+            exit 1
+          fi
+          echo "SHARD_TESTS_FILE=$SHARD_FILE" >> "$GITHUB_ENV"
+          echo "Files in shard ${{ matrix.shard }}: $(wc -l < "$SHARD_FILE")"
+
+      - name: Run Sql Endpoint Functional Tests on kernel (shard ${{ matrix.shard }})
+        run: |
+          mkdir -p logs
+          DBT_TEST_USER=notnecessaryformosttests@example.com \
+          xargs -r hatch -v run pytest \
+            --color=yes -v \
+            --profile databricks_uc_sql_endpoint \
+            -n 10 --dist=loadfile \
+            --reruns 2 --reruns-delay 120 \
+            --junitxml=logs/junit-kernel-sql-endpoint-shard-${{ matrix.shard }}.xml \
+            < "$SHARD_TESTS_FILE"
+
+      - name: Upload Kernel SQL Endpoint Test Logs
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4
+        with:
+          name: kernel-sql-endpoint-logs-shard-${{ matrix.shard }}
+          path: logs/
+          retention-days: 14
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,7 @@
 
 - Add catalogs.yml v2 support (requires `use_catalogs_v2: true` in dbt-core) ([1440](https://github.com/databricks/dbt-databricks/pull/1440))
 - Add `skip_optimize` model config to opt out of the post-materialization `OPTIMIZE` call without dropping `zorder` / `liquid_clustered_by` / `auto_liquid_cluster` from the table definition. Useful when `OPTIMIZE` is delegated to Predictive Optimization or scheduled out of band. Complements the existing run-wide `DATABRICKS_SKIP_OPTIMIZE` var by allowing project-, folder-, or model-level opt-out via standard dbt config inheritance ([#703](https://github.com/databricks/dbt-databricks/issues/703)).
+- Support the connector's Rust kernel backend via `connection_parameters: {use_kernel: true}` for SQL warehouses with PAT or Databricks OAuth auth (requires `databricks-sql-connector[kernel]` on Python 3.10+; native Azure service-principal auth is not supported) ([#1576](https://github.com/databricks/dbt-databricks/pull/1576))
 
 ### Fixes
 - Honor the `expression` field on `primary_key` constraints on the V1 materialization path. A primary key declared with `expression: RELY` (or any trailing clause) previously had its expression silently dropped. ([#1551](https://github.com/databricks/dbt-databricks/pull/1551))
@@ -20,6 +21,7 @@
 
 ### Under the Hood
 
+- Add a weekly `Kernel Integration Tests` workflow that runs the functional suite against the SQL-warehouse profile through the connector's Rust kernel backend (`DBT_DATABRICKS_USE_KERNEL=1`) (test-only, no runtime impact) ([#1576](https://github.com/databricks/dbt-databricks/pull/1576)).
 - Add functional tests for the `query` relation-config component's change handling: a streaming table's defining-query change is applied in place via `CREATE OR REFRESH`, and re-running a materialized view with an unchanged query leaves the existing relation in place instead of rebuilding it (test-only, no runtime impact).
 - Raise the `databricks-sql-connector` upper bound to `<4.3.1` to support `4.3.0` ([#1518](https://github.com/databricks/dbt-databricks/pull/1518))
 - Add a functional test for incremental column-mask removal: dropping a `column_mask` from a model with an existing incremental relation issues `ALTER COLUMN ... DROP MASK` and leaves the column unmasked (test-only, no runtime impact). ([#1514](https://github.com/databricks/dbt-databricks/pull/1514))
diff --git a/dbt/adapters/databricks/handle.py b/dbt/adapters/databricks/handle.py
@@ -10,7 +10,7 @@
 from typing import TYPE_CHECKING, Any, Optional, TypeVar
 
 from dbt.adapters.contracts.connection import AdapterResponse
-from dbt_common.exceptions import DbtRuntimeError
+from dbt_common.exceptions import DbtConfigError, DbtRuntimeError
 
 import databricks.sql as dbsql
 from databricks.sql.client import Connection, Cursor
@@ -407,16 +407,64 @@ def prepare_connection_arguments(
                 query_tags
             )
 
-        return {
+        args: dict[str, Any] = {
             "server_hostname": creds.host,
             "http_path": http_path,
-            "credentials_provider": creds_manager.credentials_provider,
             "http_headers": http_headers if http_headers else None,
             "session_configuration": session_config,
             "catalog": creds.database,
             "use_inline_params": "silent",
             "schema": creds.schema,
             "_user_agent_entry": user_agent_entry,
             "user_agent_entry": user_agent_entry,
-            **connection_parameters,
         }
+
+        if connection_parameters.get("use_kernel"):
+            SqlUtils._add_kernel_auth_arguments(args, creds)
+        else:
+            args["credentials_provider"] = creds_manager.credentials_provider
+
+        return {**args, **connection_parameters}
+
+    @staticmethod
+    def _add_kernel_auth_arguments(args: dict[str, Any], creds: DatabricksCredentials) -> None:
+        """Populate connection kwargs for the connector's Rust kernel backend.
+
+        The kernel (use_kernel=True) routes over SEA and owns the token lifecycle
+        itself, so its auth bridge rejects the connector credentials_provider that
+        dbt passes on the Thrift path. We instead forward the raw credentials the
+        bridge understands: PAT, OAuth M2M (client id + secret), or OAuth U2M
+        (browser). The branch order mirrors DatabricksCredentialManager's own auth
+        selection.
+
+        The kernel has no Azure-AD service-principal flow, so native Azure SP
+        (azure_client_id/azure_client_secret) is rejected here with a clear error
+        rather than routed to a path that can't serve it. A legacy Azure SP passed
+        via client_id/client_secret is indistinguishable from Databricks OAuth M2M
+        (the connector itself only disambiguates by trying both), so it is
+        forwarded as M2M; if it is truly Azure it fails loudly at the kernel's
+        token exchange — the same outcome as an invalid secret.
+        """
+        if creds.token:
+            args["access_token"] = creds.token
+        elif creds.azure_client_id and creds.azure_client_secret:
+            raise DbtConfigError(
+                "use_kernel=True does not support Azure service-principal auth "
+                "(azure_client_id/azure_client_secret). Use a personal access token "
+                "or Databricks OAuth (client_id/client_secret) instead, or drop "
+                "use_kernel to use the default backend."
+            )
+        elif creds.client_id and creds.client_secret:
+            # OAuth M2M
+            args["oauth_client_id"] = creds.client_id
+            args["oauth_client_secret"] = creds.client_secret
+            if creds.oauth_scopes:
+                args["oauth_scopes"] = creds.oauth_scopes
+        elif creds.auth_type == "oauth":
+            # OAuth U2M (browser). Translate dbt's auth_type "oauth" to the kernel's
+            # "databricks-oauth"; client_id is optional when using the default dbt app.
+            args["auth_type"] = "databricks-oauth"
+            if creds.client_id:
+                args["oauth_client_id"] = creds.client_id
+            if creds.oauth_scopes:
+                args["oauth_scopes"] = creds.oauth_scopes
diff --git a/pyproject.toml b/pyproject.toml
@@ -165,6 +165,7 @@ markers = [
     "external: mark test as requiring an external location",
     "python: mark test as running a python model",
     "dlt: mark test as running a DLT model",
+    "skip_kernel: skip test when running against the connector kernel backend (DBT_DATABRICKS_USE_KERNEL=1)",
 ]
 
 [tool.mypy]
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -13,13 +13,17 @@ def pytest_addoption(parser):
     parser.addoption("--profile", action="store", default=default_profile, type=str)
 
 
-# Using @pytest.mark.skip_profile('databricks_cluster') uses the 'skip_by_adapter_type'
-# autouse fixture below
+# @pytest.mark.skip_profile(...) and @pytest.mark.skip_kernel are enforced by the
+# autouse fixtures below (skip_by_profile_type, skip_on_kernel_backend).
 def pytest_configure(config):
     config.addinivalue_line(
         "markers",
         "skip_profile(profile): skip test for the given profile",
     )
+    config.addinivalue_line(
+        "markers",
+        "skip_kernel: skip test when running against the connector kernel backend",
+    )
 
 
 @pytest.fixture(scope="session")
@@ -37,6 +41,18 @@ def skip_by_profile_type(request):
                 pytest.skip(f"skipped on '{profile_type}' profile")
 
 
+@pytest.fixture(autouse=True)
+def skip_on_kernel_backend(request):
+    # The connector kernel backend (DBT_DATABRICKS_USE_KERNEL=1) routes over SEA,
+    # which is warehouse-only. Tests that pin a model to an all-purpose-cluster
+    # http_path or compute cannot connect through it, so they opt out with
+    # @pytest.mark.skip_kernel.
+    if os.environ.get("DBT_DATABRICKS_USE_KERNEL") == "1" and request.node.get_closest_marker(
+        "skip_kernel"
+    ):
+        pytest.skip("skipped on the connector kernel backend (SEA is warehouse-only)")
+
+
 # The profile dictionary, used to write out profiles.yml. It will pull in updates
 # from two separate sources, the 'profile_target' and 'profiles_config_update'.
 # The second one is useful when using alternative targets, etc.
diff --git a/tests/functional/adapter/incremental/test_incremental_column_tags.py b/tests/functional/adapter/incremental/test_incremental_column_tags.py
@@ -7,6 +7,7 @@
 from tests.functional.adapter.incremental import fixtures
 
 
+@pytest.mark.skip_kernel  # pins model http_path to a UC cluster; SEA/kernel is warehouse-only
 @pytest.mark.skip_profile("databricks_cluster")
 class TestIncrementalColumnTags(RerunSafeMixin, MaterializationV2Mixin):
     @pytest.fixture(scope="class")
@@ -57,6 +58,7 @@ def test_changing_column_tags(self, project):
 
 
 @pytest.mark.python
+@pytest.mark.skip_kernel  # Python model runs on a cluster; kernel/SEA is warehouse-only
 @pytest.mark.skip_profile("databricks_cluster")
 class TestIncrementalPythonColumnTags(TestIncrementalColumnTags):
     @pytest.fixture(scope="class")
diff --git a/tests/functional/adapter/incremental/test_incremental_strategies.py b/tests/functional/adapter/incremental/test_incremental_strategies.py
@@ -166,6 +166,7 @@ def test_incremental(self, project):
 
 
 # Only runs under SQL warehouse profile, but overrides compute at model level
+@pytest.mark.skip_kernel  # overrides compute to a UC cluster; SEA/kernel is warehouse-only
 @pytest.mark.skip_profile("databricks_uc_cluster", "databricks_cluster")
 class TestInsertOverwriteWithModelComputeOverride(IncrementalBase):
     @pytest.fixture(scope="class")
diff --git a/tests/functional/adapter/python_model/test_python_model.py b/tests/functional/adapter/python_model/test_python_model.py
@@ -182,6 +182,7 @@ def test_changing_schema_via_incremental(self, project):
 
 
 @pytest.mark.python
+@pytest.mark.skip_kernel  # pins model http_path to a cluster; SEA/kernel is warehouse-only
 @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster")
 @pytest.mark.flaky(reruns=2, reruns_delay=120)
 class TestSpecifyingHttpPath(PythonModelDataMixin, BasePythonModelTests):
diff --git a/tests/functional/adapter/spog/test_spog_connection_open.py b/tests/functional/adapter/spog/test_spog_connection_open.py
@@ -32,6 +32,7 @@ def test_probe_failure_run_succeeds(self, project):
             run_dbt(["run", "--select", "spog_smoke_model"], expect_pass=True)
 
 
+@pytest.mark.skip_kernel  # pins model to a UC cluster; SEA/kernel is warehouse-only
 @pytest.mark.skip_profile("databricks_cluster", "databricks_uc_cluster")
 class TestSpogNamedCompute:
     @pytest.fixture(scope="class")
diff --git a/tests/profiles.py b/tests/profiles.py
@@ -50,6 +50,12 @@ def _build_databricks_cluster_target(
         # If you are specifying a port for running tests, assume Docker
         # is being used and disable TLS verification
         connection_parameters["_tls_no_verify"] = True
+    # Route the whole suite through the connector's Rust kernel backend (SEA
+    # transport) when DBT_DATABRICKS_USE_KERNEL=1. Used by the weekend
+    # integration-kernel workflow. SEA is warehouse-only, so this is only
+    # meaningful on the databricks_uc_sql_endpoint profile.
+    if os.getenv("DBT_DATABRICKS_USE_KERNEL") == "1":
+        connection_parameters["use_kernel"] = True
     profile["connection_parameters"] = connection_parameters
     return profile
 
diff --git a/tests/unit/test_handle.py b/tests/unit/test_handle.py

Original file line number	Diff line number	Diff line change
`@@ -165,6 +165,7 @@ markers = [`
`165`	`165`	`"external: mark test as requiring an external location",`
`166`	`166`	`"python: mark test as running a python model",`
`167`	`167`	`"dlt: mark test as running a DLT model",`
	`168`	`+ "skip_kernel: skip test when running against the connector kernel backend (DBT_DATABRICKS_USE_KERNEL=1)",`
`168`	`169`	`]`
`169`	`170`
`170`	`171`	`[tool.mypy]`