Skip to content

Commit 70bec4d

Browse files
authored
fix: get_columns_in_relation branching for STs (#1167)
<!-- Please review our pull request review process in CONTRIBUTING.md before your proceed. --> Resolves # <!--- Include the number of the issue addressed by this PR above if applicable. Example: resolves #1234 Please review our pull request review process in CONTRIBUTING.md before your proceed. --> ### Description `compare_dbr_version` does not work for SQL warehouses (always returns 1). We should prevent `AS JSON` from running on streaming tables (supported as of 17.1) until that version becomes current for SQL warehouses ### Checklist - [ ] I have run this code in development and it appears to resolve the stated issue - [ ] This PR includes tests, or tests are not required/relevant for this PR - [ ] I have updated the `CHANGELOG.md` and added information about my change to the "dbt-databricks next" section.
1 parent 7f1ae0e commit 70bec4d

3 files changed

Lines changed: 7 additions & 39 deletions

File tree

CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
- Improve ANSI mode error handling for Python models and add debug instrumentation ([1157](https://github.com/databricks/dbt-databricks/pull/1157))
66
- Remove external path on intermediate tables for incremental models (with Materialization V2) ([1161](https://github.com/databricks/dbt-databricks/pull/1161))
7+
- Fix get_columns_in_relation branching logic for streaming tables to prevent it from running `AS JSON`
78

89
## dbt-databricks 1.10.10 (August 20, 2025)
910

dbt/adapters/databricks/impl.py

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -485,10 +485,9 @@ def get_columns_in_relation( # type: ignore[override]
485485
relation.is_hive_metastore()
486486
or self.compare_dbr_version(16, 2) < 0
487487
or relation.type == DatabricksRelationType.MaterializedView
488-
or (
489-
relation.type == DatabricksRelationType.StreamingTable
490-
and self.compare_dbr_version(17, 1) < 0
491-
)
488+
# TODO: Replace with self.compare_dbr_version(17, 1) < 0 when 17.1 is current version
489+
# for SQL warehouses
490+
or relation.type == DatabricksRelationType.StreamingTable
492491
)
493492
return self.get_column_behavior.get_columns_in_relation(self, relation, use_legacy_logic)
494493

tests/unit/test_adapter.py

Lines changed: 3 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1095,11 +1095,12 @@ def test_get_columns_new_logic(self, mock_get_columns, adapter, unity_relation):
10951095
assert result[0].dtype == "string"
10961096
assert result[0].comment == "comment1"
10971097

1098+
@pytest.mark.parametrize("compare_dbr_version", [-1, 1])
10981099
@patch(
10991100
"dbt.adapters.databricks.behaviors.columns.GetColumnsByDescribe._get_columns_with_comments"
11001101
)
11011102
def test_get_columns_streaming_table_legacy_logic(
1102-
self, mock_get_columns, adapter, unity_relation
1103+
self, mock_get_columns, adapter, unity_relation, compare_dbr_version
11031104
):
11041105
streaming_relation = DatabricksRelation.create(
11051106
database=unity_relation.database,
@@ -1108,7 +1109,7 @@ def test_get_columns_streaming_table_legacy_logic(
11081109
type=DatabricksRelation.StreamingTable,
11091110
)
11101111
# Return value less than 0 means version is older than 17.1
1111-
with patch.object(adapter, "compare_dbr_version", return_value=-1):
1112+
with patch.object(adapter, "compare_dbr_version", return_value=compare_dbr_version):
11121113
mock_get_columns.return_value = [
11131114
{"col_name": "stream_col", "data_type": "int", "comment": "streaming col"},
11141115
]
@@ -1119,39 +1120,6 @@ def test_get_columns_streaming_table_legacy_logic(
11191120
assert result[0].dtype == "int"
11201121
assert result[0].comment == "streaming col"
11211122

1122-
@patch(
1123-
"dbt.adapters.databricks.behaviors.columns.GetColumnsByDescribe._get_columns_with_comments"
1124-
)
1125-
def test_get_columns_streaming_table_new_logic(self, mock_get_columns, adapter, unity_relation):
1126-
streaming_relation = DatabricksRelation.create(
1127-
database=unity_relation.database,
1128-
schema=unity_relation.schema,
1129-
identifier=unity_relation.identifier,
1130-
type=DatabricksRelation.StreamingTable,
1131-
)
1132-
# Return value 0 means version is 17.1
1133-
with patch.object(adapter, "compare_dbr_version", return_value=0):
1134-
json_data = """
1135-
{
1136-
"columns": [
1137-
{
1138-
"name": "stream_col",
1139-
"type": {"name": "int"},
1140-
"comment": "streaming col"
1141-
}
1142-
]
1143-
}
1144-
"""
1145-
mock_get_columns.return_value = [{"json_metadata": json_data}]
1146-
result = adapter.get_columns_in_relation(streaming_relation)
1147-
mock_get_columns.assert_called_with(
1148-
adapter, streaming_relation, "get_columns_comments_as_json"
1149-
)
1150-
assert len(result) == 1
1151-
assert result[0].column == "stream_col"
1152-
assert result[0].dtype == "int"
1153-
assert result[0].comment == "streaming col"
1154-
11551123
@patch(
11561124
"dbt.adapters.databricks.behaviors.columns.GetColumnsByDescribe._get_columns_with_comments"
11571125
)

0 commit comments

Comments
 (0)