Revert REPLACE USING usage in insert overwrite (databricks#1138)

Author: ericj-db

CHANGELOG.md

@@ -3,6 +3,9 @@
 ### Features
 - Support column tags for views using `ALTER TABLE`
 
+### Under the hood
+- Revert `REPLACE USING` syntax being used for insert overwrite ([1025](https://github.com/databricks/dbt-databricks/issues/1025))
+
 ## dbt-databricks 1.10.8 (August 4, 2025)
 
 ### Features

dbt/include/databricks/macros/materializations/incremental/incremental.sql

@@ -52,7 +52,7 @@
     {%- else -%}
       {{ log("Existing relation found, proceeding with incremental work")}}
       {#-- Set Overwrite Mode to DYNAMIC for subsequent incremental operations --#}
-      {%- if incremental_strategy == 'insert_overwrite' and partition_by and adapter.compare_dbr_version(16, 3) < 0 -%}
+      {%- if incremental_strategy == 'insert_overwrite' and partition_by -%}
         {{ set_overwrite_mode('DYNAMIC') }}
       {%- endif -%}
       {#-- Relation must be merged --#}
@@ -118,7 +118,7 @@
       {% do persist_docs(target_relation, model, for_relation=language=='python') %}
     {%- else -%}
       {#-- Set Overwrite Mode to DYNAMIC for subsequent incremental operations --#}
-      {%- if incremental_strategy == 'insert_overwrite' and partition_by and adapter.compare_dbr_version(16, 3) < 0 -%}
+      {%- if incremental_strategy == 'insert_overwrite' and partition_by -%}
         {{ set_overwrite_mode('DYNAMIC') }}
       {%- endif -%}
       {#-- Relation must be merged --#}
@@ -177,7 +177,7 @@
     {{ run_hooks(post_hooks) }}
   {%- endif -%}
 
-  {%- if incremental_strategy == 'insert_overwrite' and not full_refresh and adapter.compare_dbr_version(16, 3) < 0 -%}
+  {%- if incremental_strategy == 'insert_overwrite' and not full_refresh -%}
     {{ set_overwrite_mode('STATIC') }}
   {%- endif -%}
 
@@ -191,7 +191,7 @@
       set spark.sql.sources.partitionOverwriteMode = {{ value }}
     {%- endcall -%}
   {% else %}
-    {{ exceptions.warn("INSERT OVERWRITE is supported on SQL warehouses with DBR 16.3+. On older DBR versions, this strategy would be equivalent to using the table materialization.") }}
+    {{ exceptions.warn("INSERT OVERWRITE is only properly supported on all-purpose clusters.  On SQL Warehouses, this strategy would be equivalent to using the table materialization.") }}
   {% endif %}
 {% endmacro %}
 
@@ -216,4 +216,4 @@
     {%- set configuration_changes = model_config.get_changeset(existing_config) -%}
     {{ apply_config_changeset(target_relation, model, configuration_changes) }}
   {% endif %}
-{% endmacro %}
+{% endmacro %}

dbt/include/databricks/macros/materializations/incremental/strategies.sql

@@ -38,32 +38,9 @@
     {%- endfor -%}
     {%- set dest_cols_csv = dest_columns | join(', ') -%}
     {%- set source_cols_csv = common_columns | join(', ') -%}
-    
-    {%- if adapter.compare_dbr_version(16, 3) >= 0 -%}
-        {{ get_insert_replace_using_sql(source_relation, target_relation, source_cols_csv) }}
-    {%- else -%}
-        {#-- Use traditional INSERT OVERWRITE for older DBR versions --#}
-        insert overwrite table {{ target_relation }}
-        {{ partition_cols(label="partition") }}
-        select {{ source_cols_csv }} from {{ source_relation }}
-    {%- endif -%}
-{% endmacro %}
-
-{% macro get_insert_replace_using_sql(source_relation, target_relation, source_cols_csv) %}
-    {%- set partition_by = config.get('partition_by') -%}
-    {%- if partition_by -%}
-        {%- if partition_by is string -%}
-            {%- set partition_by = [partition_by] -%}
-        {%- endif -%}
-        {%- set partition_cols_csv = partition_by | join(', ') -%}
-        insert into table {{ target_relation }}
-        replace using ({{ partition_cols_csv }})
-        select {{ source_cols_csv }} from {{ source_relation }}
-    {%- else -%}
-        {#-- Fallback to regular insert if no partitions defined --#}
-        insert overwrite table {{ target_relation }}
-        select {{ source_cols_csv }} from {{ source_relation }}
-    {%- endif -%}
+    insert overwrite table {{ target_relation }}
+    {{ partition_cols(label="partition") }}
+    select {{source_cols_csv}} from {{ source_relation }}
 {% endmacro %}
 
 {% macro get_replace_where_sql(args_dict) -%}

dbt/include/databricks/macros/materializations/incremental/validate.sql

@@ -24,6 +24,12 @@
     You can only choose this strategy when file_format is set to 'delta'
   {%- endset %}
 
+  {% set invalid_insert_overwrite_endpoint_msg -%}
+    Invalid incremental strategy provided: {{ raw_strategy }}
+    You cannot use this strategy when connecting via warehouse
+    Use the 'merge' or 'replace_where' strategy instead
+  {%- endset %}
+
   {% if raw_strategy not in adapter.valid_incremental_strategies() %}
     {{ log("WARNING - You are using an unsupported incremental strategy: " ~ raw_strategy) }}
     {{ log("You can ignore this warning if you are using a custom incremental strategy") }}

tests/functional/adapter/incremental/test_incremental_strategies.py

@@ -70,6 +70,7 @@ def project_config_update(self):
         }
 
 
+@pytest.mark.skip_profile("databricks_uc_sql_endpoint")
 class InsertOverwriteBase(IncrementalBase):
     @pytest.fixture(scope="class")
     def seeds(self):
@@ -92,10 +93,12 @@ def test_incremental(self, project):
         util.check_relations_equal(project.adapter, ["overwrite_model", "overwrite_expected"])
 
 
+@pytest.mark.skip_profile("databricks_uc_sql_endpoint")
 class TestInsertOverwriteDelta(InsertOverwriteBase):
     pass
 
 
+@pytest.mark.skip_profile("databricks_uc_sql_endpoint")
 class TestInsertOverwriteWithPartitionsDelta(InsertOverwriteBase):
     @pytest.fixture(scope="class")
     def project_config_update(self):
@@ -117,6 +120,7 @@ def test_incremental(self, project):
         util.check_relations_equal(project.adapter, ["overwrite_model", "upsert_expected"])
 
 
+@pytest.mark.skip_profile("databricks_uc_sql_endpoint")
 class TestInsertOverwriteChangeSchema(InsertOverwriteBase):
     @pytest.fixture(scope="class")
     def models(self):
@@ -174,6 +178,37 @@ def test_incremental(self, project):
         util.check_relations_equal(project.adapter, ["overwrite_model", "upsert_expected"])
 
 
+# Insert overwrite in SQL warehouse is expected to behave like a table materialization
+# We support this as a short term hack for customers who want the side effect of reusing
+# the same table on subsequent runs
+@pytest.mark.skip_profile("databricks_uc_cluster", "databricks_cluster")
+class TestInsertOverwriteSqlWarehouse(IncrementalBase):
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {
+            "models": {
+                "+incremental_strategy": "insert_overwrite",
+                "+partition_by": "id",
+            },
+        }
+
+    @pytest.fixture(scope="class")
+    def seeds(self):
+        return {
+            "overwrite_expected.csv": fixtures.overwrite_expected,
+        }
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "overwrite_model.sql": fixtures.base_model,
+        }
+
+    def test_incremental(self, project):
+        self.seed_and_run_twice()
+        util.check_relations_equal(project.adapter, ["overwrite_model", "overwrite_expected"])
+
+
 @pytest.mark.external
 @pytest.mark.skip("This test is not repeatable due to external location")
 class TestInsertOverwriteParquet(InsertOverwriteBase):