Skip to content

Commit 192317e

Browse files
authored
fix: return True from has_dbr_capability parse stub on SQL warehouses (#1331) (#1449)
Motivated by [#1331](#1331). The class-level `@available.parse(lambda *a, **k: False)` stub on `has_dbr_capability` returns a constant `False` at parse time, so every macro branching on a capability takes the legacy path during parse/compile — even on SQL warehouses, which support all capabilities we currently ship. This is a 1.11.0 regression: the pre-1.11 stub (`compare_dbr_version`) returned `0`, and `0 >= 0` accidentally selected the modern path. ## Fix Shadow `_parse_replacements_` at adapter `__init__` with a bound method that consults the profile's `http_path`: - **SQL warehouse** → `True` for capabilities with `sql_warehouse_supported=True`. - **Cluster** → `False` (real DBR version is only known at runtime). Cluster users on supported DBR are still affected — fully fixing that requires a larger change. This PR is the contained warehouse-only fix. ## Test plan - [x] `hatch run pytest tests/unit/test_adapter_capabilities.py -v` — 19 pass (4 new). - [x] `hatch run pre-commit run --all-files` — clean. - [ ] CI functional run.
1 parent 42431d5 commit 192317e

4 files changed

Lines changed: 106 additions & 1 deletion

File tree

CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
### Fixes
88

99
- Fix missing f-string prefix in `JobRunsApi.submit` debug log ([#1471](https://github.com/databricks/dbt-databricks/pull/1471))
10+
- Fix capability-branching macros falling through to their legacy path at parse/compile time on SQL warehouses. The parse-time stub of `has_dbr_capability` now returns `True` on warehouse profiles for capabilities flagged `sql_warehouse_supported`, so macros select the modern branch during compilation instead of the legacy fallback. ([#1449](https://github.com/databricks/dbt-databricks/pull/1449) closes [#1331](https://github.com/databricks/dbt-databricks/issues/1331))
1011

1112
### Under the Hood
1213

dbt/adapters/databricks/impl.py

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@
5151
DatabricksConnectionManager,
5252
DatabricksDBTConnection,
5353
)
54-
from dbt.adapters.databricks.dbr_capabilities import DBRCapability
54+
from dbt.adapters.databricks.dbr_capabilities import DBRCapabilities, DBRCapability
5555
from dbt.adapters.databricks.global_state import GlobalState
5656
from dbt.adapters.databricks.handle import SqlUtils
5757
from dbt.adapters.databricks.python_models.python_submissions import (
@@ -90,6 +90,7 @@
9090
from dbt.adapters.databricks.utils import (
9191
get_first_row,
9292
handle_missing_objects,
93+
is_cluster_http_path,
9394
quote,
9495
redact_credentials,
9596
)
@@ -269,6 +270,25 @@ def __init__(self, config: Any, mp_context: SpawnContext) -> None:
269270
self.get_behavior_flag_no_warn(USE_MANAGED_ICEBERG["name"])
270271
)
271272

273+
# Warehouses always meet capability cutoffs at parse time; clusters keep the
274+
# conservative False until a real connection is available.
275+
# `_parse_replacements_` is injected by AdapterMeta, so mypy can't resolve it here.
276+
self._parse_replacements_ = {
277+
**self.__class__._parse_replacements_, # type: ignore[has-type]
278+
"has_dbr_capability": self._has_dbr_capability_parse,
279+
}
280+
281+
def _has_dbr_capability_parse(self, capability_name: str) -> bool:
282+
"""Parse-time stub: True only on SQL warehouses for warehouse-supported capabilities."""
283+
creds = self.config.credentials
284+
if is_cluster_http_path(creds.http_path, creds.cluster_id):
285+
return False
286+
try:
287+
capability = DBRCapability(capability_name.lower())
288+
except ValueError:
289+
return False
290+
return DBRCapabilities(is_sql_warehouse=True).has_capability(capability)
291+
272292
@property
273293
def _behavior_flags(self) -> list[BehaviorFlag]:
274294
return [
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import pytest
2+
from dbt.tests import util
3+
4+
# Exercises the parse-time stub for `has_dbr_capability` on SQL warehouses.
5+
#
6+
# `databricks__dateadd` branches on `has_dbr_capability('timestampdiff')`. If the
7+
# parse stub returned False, the legacy `spark__dateadd` path is selected at compile
8+
# time and raises a compile error on uppercase dateparts. With the warehouse stub
9+
# returning True, the modern `timestampadd` path is selected and the run succeeds.
10+
model_sql = """
11+
select {{ dateadd('DAY', 1, "cast('2024-01-01' as timestamp)") }} as ts
12+
"""
13+
14+
15+
class TestDateAddUppercaseDatepartOnWarehouse:
16+
@pytest.fixture(scope="class")
17+
def models(self):
18+
return {"uppercase_dateadd.sql": model_sql}
19+
20+
@pytest.mark.skip_profile("databricks_uc_cluster", "databricks_cluster")
21+
def test_uppercase_datepart_compiles_and_runs(self, project):
22+
util.run_dbt(["run"])

tests/unit/test_adapter_capabilities.py

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,3 +200,65 @@ def record(event_msg):
200200
"supports(MicrobatchConcurrency) must not fire BehaviorChangeEvent "
201201
f"(fired {len(caught)} times)"
202202
)
203+
204+
205+
class TestHasDbrCapabilityParseStub:
206+
"""Parse-time stub for has_dbr_capability."""
207+
208+
def _make_adapter(self, http_path: str) -> DatabricksAdapter:
209+
project_cfg = {
210+
"name": "test_project",
211+
"version": "0.1",
212+
"profile": "test",
213+
"project-root": "/tmp/dbt/does-not-exist",
214+
"config-version": 2,
215+
}
216+
profile_cfg = {
217+
"outputs": {
218+
"test": {
219+
"type": "databricks",
220+
"catalog": "main",
221+
"schema": "analytics",
222+
"host": "test.databricks.com",
223+
"http_path": http_path,
224+
"token": "dapi" + "X" * 32,
225+
}
226+
},
227+
"target": "test",
228+
}
229+
config = config_from_parts_or_dicts(project_cfg, profile_cfg)
230+
return DatabricksAdapter(config, get_context("spawn"))
231+
232+
@pytest.fixture
233+
def cluster_adapter(self) -> DatabricksAdapter:
234+
return self._make_adapter("sql/protocolv1/o/1234567890123456/1234-567890-test123")
235+
236+
@pytest.fixture
237+
def warehouse_adapter(self) -> DatabricksAdapter:
238+
return self._make_adapter("/sql/1.0/warehouses/abc123")
239+
240+
def test_warehouse_returns_true_for_sql_warehouse_supported_capability(self, warehouse_adapter):
241+
stub = warehouse_adapter._parse_replacements_["has_dbr_capability"]
242+
assert stub("timestampdiff") is True
243+
assert stub("insert_by_name") is True
244+
assert stub("comment_on_column") is True
245+
assert stub("replace_on") is True
246+
assert stub("iceberg") is True
247+
248+
def test_warehouse_returns_false_for_unsupported_capability(self, warehouse_adapter):
249+
# STREAMING_TABLE_JSON_METADATA is cluster-only (sql_warehouse_supported=False).
250+
stub = warehouse_adapter._parse_replacements_["has_dbr_capability"]
251+
assert stub("streaming_table_json_metadata") is False
252+
253+
def test_cluster_returns_false_for_all_capabilities(self, cluster_adapter):
254+
stub = cluster_adapter._parse_replacements_["has_dbr_capability"]
255+
assert stub("timestampdiff") is False
256+
assert stub("insert_by_name") is False
257+
assert stub("comment_on_column") is False
258+
assert stub("streaming_table_json_metadata") is False
259+
260+
def test_unknown_capability_returns_false(self, warehouse_adapter, cluster_adapter):
261+
# Runtime method raises ValueError; parse stub silently returns False.
262+
for adapter in (warehouse_adapter, cluster_adapter):
263+
stub = adapter._parse_replacements_["has_dbr_capability"]
264+
assert stub("not_a_real_capability") is False

0 commit comments

Comments
 (0)