Skip to content

Commit e318a44

Browse files
ajhlee-dbtclaude
andauthored
feat: Add instrumentation for recording add_query, get_relation_config, is_uniform, has_dbr_capability, and is_cluster methods (#1508)
<!-- Please review our pull request review process in CONTRIBUTING.md before your proceed. --> Resolves #N/A. <!--- Include the number of the issue addressed by this PR above if applicable. Example: resolves #1234 Please review our pull request review process in CONTRIBUTING.md before your proceed. --> ### Description Instruments the following adapter functions for recording: - `has_dbr_capability` and `is_cluster` (via the auto record decorator) - `add_query` - `get_relation_config` - `is_uniform` ### Checklist - [X] I have run this code in development and it appears to resolve the stated issue - [X] This PR includes tests, or tests are not required/relevant for this PR - [ ] I have updated the `CHANGELOG.md` and added information about my change to the "dbt-databricks next" section. --------- Signed-off-by: Anna Lee <anna.lee@dbtlabs.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 93d6314 commit e318a44

5 files changed

Lines changed: 839 additions & 0 deletions

File tree

dbt/adapters/databricks/impl.py

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
from dbt_common.behavior_flags import BehaviorFlag
3333
from dbt_common.contracts.config.base import BaseConfig, MergeBehavior
3434
from dbt_common.exceptions import DbtConfigError, DbtInternalError, DbtRuntimeError
35+
from dbt_common.record import auto_record_function, record_function
3536
from dbt_common.utils import executor
3637
from dbt_common.utils.dict import AttrDict
3738
from packaging import version
@@ -62,6 +63,11 @@
6263
ServerlessClusterPythonJobHelper,
6364
WorkflowPythonJobHelper,
6465
)
66+
from dbt.adapters.databricks.record.record_types import (
67+
DatabricksAdapterAddQueryRecord,
68+
DatabricksAdapterGetRelationConfigRecord,
69+
DatabricksAdapterIsUniformRecord,
70+
)
6571
from dbt.adapters.databricks.relation import (
6672
KEY_TABLE_PROVIDER,
6773
DatabricksRelation,
@@ -332,6 +338,12 @@ def quote(self, identifier): # type: ignore[override,no-untyped-def]
332338
return quote(identifier)
333339

334340
@available.parse(lambda *a, **k: 0)
341+
@record_function(
342+
DatabricksAdapterIsUniformRecord,
343+
method=True,
344+
index_on_thread_id=True,
345+
id_field_name="thread_id",
346+
)
335347
def is_uniform(self, config: BaseConfig) -> bool:
336348
catalog_relation: DatabricksCatalogRelation = self.build_catalog_relation(config.model) # type:ignore
337349
if catalog_relation.table_format == constants.ICEBERG_TABLE_FORMAT:
@@ -425,6 +437,7 @@ def has_capability(self, capability: DBRCapability) -> bool:
425437
return conn.has_capability(capability)
426438

427439
@available.parse(lambda *a, **k: False)
440+
@auto_record_function("AdapterHasDbrCapability", group="Available")
428441
def has_dbr_capability(self, capability_name: str) -> bool:
429442
"""
430443
Check if a DBR capability is available for current compute.
@@ -889,6 +902,13 @@ def get_behavior_flag_no_warn(self, behavior_flag_name: str) -> bool:
889902
behavior_flag = getattr(self.behavior, behavior_flag_name)
890903
return behavior_flag.no_warn
891904

905+
@available.parse(lambda *a, **k: (None, None))
906+
@record_function(
907+
DatabricksAdapterAddQueryRecord,
908+
method=True,
909+
index_on_thread_id=True,
910+
id_field_name="thread_id",
911+
)
892912
def add_query(
893913
self,
894914
sql: str,
@@ -1059,6 +1079,12 @@ def parse_columns_and_constraints(
10591079
return enriched_columns, parsed_constraints
10601080

10611081
@available.parse(lambda *a, **k: {})
1082+
@record_function(
1083+
DatabricksAdapterGetRelationConfigRecord,
1084+
method=True,
1085+
index_on_thread_id=True,
1086+
id_field_name="thread_id",
1087+
)
10621088
def get_relation_config(
10631089
self,
10641090
relation: DatabricksRelation,
@@ -1097,6 +1123,7 @@ def get_config_from_model(self, model: RelationConfig) -> DatabricksRelationConf
10971123
)
10981124

10991125
@available
1126+
@auto_record_function("AdapterIsCluster", group="Available")
11001127
def is_cluster(self) -> bool:
11011128
"""Check if the current connection is a cluster."""
11021129
return self.connections.is_cluster()
Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
import dataclasses
2+
import json
3+
from typing import Any, Optional
4+
5+
from dbt.adapters.record.serialization import serialize_bindings
6+
from dbt_common.record import Record, Recorder
7+
8+
from dbt.adapters.databricks.relation import DatabricksRelation
9+
from dbt.adapters.databricks.relation_configs.base import DatabricksRelationConfigBase
10+
11+
12+
def _serialize_component(component: Any) -> dict[str, Any]:
13+
"""Serialize a relation config component to a JSON-native dict.
14+
15+
pydantic v2 exposes ``model_dump(mode="json")``; v1 only has ``json()``,
16+
which is round-tripped to coerce tuples, sets, and enums to JSON types.
17+
Under v1 the ``model_config`` ConfigDict is treated as a regular field and
18+
leaks into the output, so it is dropped to match the v2 shape.
19+
"""
20+
if hasattr(component, "model_dump"):
21+
return component.model_dump(mode="json")
22+
serialized = json.loads(component.json())
23+
serialized.pop("model_config", None)
24+
return serialized
25+
26+
27+
@dataclasses.dataclass
28+
class DatabricksAdapterAddQueryParams:
29+
thread_id: str
30+
sql: str
31+
auto_begin: bool = True
32+
bindings: Optional[Any] = None
33+
abridge_sql_log: bool = False
34+
close_cursor: bool = False
35+
36+
def _to_dict(self) -> dict[str, Any]:
37+
return {
38+
"thread_id": self.thread_id,
39+
"sql": self.sql,
40+
"auto_begin": self.auto_begin,
41+
"bindings": serialize_bindings(self.bindings),
42+
"abridge_sql_log": self.abridge_sql_log,
43+
"close_cursor": self.close_cursor,
44+
}
45+
46+
47+
@dataclasses.dataclass
48+
class DatabricksAdapterAddQueryResult:
49+
return_val: tuple
50+
51+
def _to_dict(self) -> dict[str, Any]:
52+
return {
53+
"return_val": {
54+
"conn": "conn",
55+
"cursor": "cursor",
56+
}
57+
}
58+
59+
60+
@Recorder.register_record_type
61+
class DatabricksAdapterAddQueryRecord(Record):
62+
"""Implements record/replay support for the DatabricksAdapter.add_query() method."""
63+
64+
params_cls = DatabricksAdapterAddQueryParams
65+
result_cls = DatabricksAdapterAddQueryResult
66+
group = "Available"
67+
68+
69+
@dataclasses.dataclass
70+
class DatabricksAdapterIsUniformParams:
71+
thread_id: str
72+
config: Any
73+
74+
def _to_dict(self) -> dict[str, Any]:
75+
return {
76+
"thread_id": self.thread_id,
77+
"config_model": self.config.model.to_dict(),
78+
"materialized": self.config.get("materialized"),
79+
}
80+
81+
82+
@dataclasses.dataclass
83+
class DatabricksAdapterIsUniformResult:
84+
return_val: bool
85+
86+
87+
@Recorder.register_record_type
88+
class DatabricksAdapterIsUniformRecord(Record):
89+
"""Implements record/replay support for the DatabricksAdapter.is_uniform() method."""
90+
91+
params_cls = DatabricksAdapterIsUniformParams
92+
result_cls = DatabricksAdapterIsUniformResult
93+
group = "Available"
94+
95+
96+
@dataclasses.dataclass
97+
class DatabricksAdapterGetRelationConfigParams:
98+
thread_id: str
99+
relation: DatabricksRelation
100+
model_config: Optional[DatabricksRelationConfigBase]
101+
102+
def _to_dict(self) -> dict[str, Any]:
103+
from dbt.adapters.record.serialization import serialize_base_relation
104+
105+
return {
106+
"thread_id": self.thread_id,
107+
"relation": serialize_base_relation(self.relation),
108+
"model_config": (
109+
{
110+
"config": {
111+
k: _serialize_component(v) for k, v in self.model_config.config.items()
112+
}
113+
}
114+
if self.model_config
115+
else None
116+
),
117+
}
118+
119+
120+
@dataclasses.dataclass
121+
class DatabricksAdapterGetRelationConfigResult:
122+
return_val: Optional[dict] # DatabricksRelationConfigBase serialized as dict, or None
123+
124+
def _to_dict(self) -> dict[str, Any]:
125+
# return_val is already converted to dict by the constructor
126+
return {
127+
"return_val": self.return_val,
128+
}
129+
130+
def __init__(self, return_val: Optional[Any]) -> None:
131+
if return_val is not None and not isinstance(return_val, dict):
132+
self.return_val = {
133+
"config": {k: _serialize_component(v) for k, v in return_val.config.items()}
134+
}
135+
else:
136+
self.return_val = return_val
137+
138+
@classmethod
139+
def _from_dict(cls, data: dict[str, Any]) -> "DatabricksAdapterGetRelationConfigResult":
140+
return cls(return_val=data.get("return_val"))
141+
142+
143+
@Recorder.register_record_type
144+
class DatabricksAdapterGetRelationConfigRecord(Record):
145+
"""Implements record/replay support for the DatabricksAdapter.get_relation_config() method."""
146+
147+
params_cls = DatabricksAdapterGetRelationConfigParams
148+
result_cls = DatabricksAdapterGetRelationConfigResult
149+
group = "Available"

tests/unit/record/__init__.py

Whitespace-only changes.
Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
{
2+
"database": "test_schema",
3+
"schema": "test_schema",
4+
"name": "m_table",
5+
"resource_type": "model",
6+
"package_name": "test_project",
7+
"path": "m_table.sql",
8+
"original_file_path": "models/m_table.sql",
9+
"unique_id": "model.test_project.m_table",
10+
"fqn": [
11+
"test_project",
12+
"m_table"
13+
],
14+
"alias": "m_table",
15+
"checksum": {
16+
"name": "sha256",
17+
"checksum": ""
18+
},
19+
"config": {
20+
"enabled": true,
21+
"alias": null,
22+
"schema": null,
23+
"database": null,
24+
"tags": [],
25+
"meta": {},
26+
"group": null,
27+
"materialized": "table",
28+
"incremental_strategy": null,
29+
"batch_size": null,
30+
"lookback": 1,
31+
"begin": null,
32+
"persist_docs": {},
33+
"post-hook": [],
34+
"pre-hook": [],
35+
"quoting": {},
36+
"column_types": {},
37+
"full_refresh": null,
38+
"unique_key": null,
39+
"on_schema_change": "ignore",
40+
"on_configuration_change": "apply",
41+
"grants": {},
42+
"packages": [],
43+
"docs": {
44+
"show": true,
45+
"node_color": null
46+
},
47+
"contract": {
48+
"enforced": false,
49+
"alias_types": true
50+
},
51+
"event_time": null,
52+
"concurrent_batches": null,
53+
"access": "protected",
54+
"freshness": null
55+
},
56+
"tags": [],
57+
"description": "",
58+
"columns": {
59+
"id": {
60+
"name": "id",
61+
"description": "",
62+
"meta": {},
63+
"data_type": null,
64+
"constraints": [],
65+
"quote": null,
66+
"config": {
67+
"meta": {},
68+
"tags": []
69+
},
70+
"tags": [],
71+
"granularity": null,
72+
"doc_blocks": []
73+
}
74+
},
75+
"meta": {},
76+
"group": null,
77+
"docs": {
78+
"show": true,
79+
"node_color": null
80+
},
81+
"patch_path": "test_project://models/schema.yml",
82+
"build_path": null,
83+
"unrendered_config": {
84+
"materialized": "table"
85+
},
86+
"created_at": 0,
87+
"config_call_dict": {
88+
"materialized": "table"
89+
},
90+
"unrendered_config_call_dict": {},
91+
"relation_name": "`test_schema`.`test_schema`.`m_table`",
92+
"raw_code": "{{ config(materialized='table') }}\nselect 1 as id, 'a' as val",
93+
"doc_blocks": [],
94+
"language": "sql",
95+
"refs": [],
96+
"sources": [],
97+
"metrics": [],
98+
"functions": [],
99+
"depends_on": {
100+
"macros": [],
101+
"nodes": []
102+
},
103+
"compiled_path": "target/compiled/test_project/models/m_table.sql",
104+
"compiled": true,
105+
"compiled_code": "\nselect 1 as id, 'a' as val",
106+
"extra_ctes_injected": true,
107+
"extra_ctes": [],
108+
"contract": {
109+
"enforced": false,
110+
"alias_types": true,
111+
"checksum": null
112+
},
113+
"access": "protected",
114+
"constraints": [],
115+
"version": null,
116+
"latest_version": null,
117+
"deprecation_date": null,
118+
"defer_relation": null,
119+
"primary_key": [],
120+
"time_spine": null,
121+
"batch": null
122+
}

0 commit comments

Comments
 (0)