Skip to content

Commit b182e8b

Browse files
authored
test: add incremental full-refresh recreate and strategy-validation functional tests (#1524)
Adds functional coverage for two incremental-materialization behaviors on the `databricks_uc_sql_endpoint` profile. 1. **`--full-refresh` recreate of an existing incremental relation** (appended to `test_incremental_replace_table.py`, V1 + V2). The existing full-refresh-over-existing test is `skip_profile`'d off the SQL-warehouse profile and has no V2 variant. The new tests use the append strategy so the second run accumulates rows the create branch never emits; `--full-refresh` erases them, proving the relation is recreated (server-observable via the resulting row set). 2. **Incremental-strategy validation errors** (new `test_incremental_validation_errors.py`): merge on a non-delta `file_format`, `delete+insert` without `unique_key`, and `merge_update_columns` + `merge_exclude_columns` together — each asserts the run fails. These validation paths had no functional or unit coverage. All 5 tests pass on `databricks_uc_sql_endpoint`; no existing test was changed (the only edit to a pre-existing file is an additive append).
1 parent f40989e commit b182e8b

2 files changed

Lines changed: 126 additions & 0 deletions

File tree

tests/functional/adapter/incremental/test_incremental_replace_table.py

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,50 @@ def test_replace(self, project):
2525
util.write_file(fixtures.replace_incremental, "models", "model.sql")
2626
util.run_dbt(["run", "--full-refresh"])
2727
util.check_relations_equal(project.adapter, ["model", "seed"])
28+
29+
30+
class FullRefreshRecreateBase(RerunSafeMixin):
31+
"""`--full-refresh` recreates an existing incremental relation from its create
32+
branch. Append accumulates extra rows first, so the erased rows prove the recreate."""
33+
34+
@pytest.fixture(scope="class")
35+
def relations_to_reset(self):
36+
return ("full_refresh_recreate",)
37+
38+
@pytest.fixture(scope="class")
39+
def models(self):
40+
return {"full_refresh_recreate.sql": fixtures.base_model}
41+
42+
def test_full_refresh_recreates_existing_relation(self, project):
43+
# Create {1: hello, 2: goodbye}.
44+
util.run_dbt(["run"])
45+
# Incremental (append) run accumulates {2: yo, 3: anyway}.
46+
util.run_dbt(["run"])
47+
accumulated = project.run_sql(
48+
"select id, msg from full_refresh_recreate order by id, msg", fetch="all"
49+
)
50+
# Four rows incl. duplicate id=2 — there is real state for a recreate to erase.
51+
assert accumulated == [(1, "hello"), (2, "goodbye"), (2, "yo"), (3, "anyway")]
52+
53+
# --full-refresh renders the non-incremental branch and recreates the table.
54+
util.run_dbt(["run", "--full-refresh"])
55+
recreated = project.run_sql(
56+
"select id, msg from full_refresh_recreate order by id, msg", fetch="all"
57+
)
58+
# Only the create-branch rows remain; the accumulated rows are gone.
59+
assert recreated == [(1, "hello"), (2, "goodbye")]
60+
61+
62+
class TestIncrementalFullRefreshRecreate(FullRefreshRecreateBase):
63+
@pytest.fixture(scope="class")
64+
def project_config_update(self):
65+
return {"models": {"+incremental_strategy": "append"}}
66+
67+
68+
class TestIncrementalFullRefreshRecreateV2(FullRefreshRecreateBase):
69+
@pytest.fixture(scope="class")
70+
def project_config_update(self):
71+
return {
72+
"flags": {"use_materialization_v2": True},
73+
"models": {"+incremental_strategy": "append"},
74+
}
Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
"""Incremental-strategy validation errors raised by validate.sql / strategies.sql.
2+
3+
Each fires as a compiler error during the run; the proof is the asserted run failure.
4+
"""
5+
6+
import pytest
7+
from dbt.tests import util
8+
9+
from tests.functional.adapter.fixtures import RerunSafeMixin
10+
from tests.functional.adapter.incremental import fixtures
11+
12+
13+
class TestMergeOnNonDeltaFormatRaises:
14+
"""merge on a non-delta/hudi file_format -> dbt_databricks_validate_get_incremental_strategy
15+
raises before any DDL (validate.sql, the merge branch)."""
16+
17+
@pytest.fixture(scope="class")
18+
def project_config_update(self):
19+
return {
20+
"models": {
21+
"+incremental_strategy": "merge",
22+
"+unique_key": "id",
23+
"+file_format": "parquet",
24+
}
25+
}
26+
27+
@pytest.fixture(scope="class")
28+
def models(self):
29+
return {"merge_non_delta.sql": fixtures.base_model}
30+
31+
def test_merge_on_parquet_raises(self, project):
32+
util.run_dbt(["run"], expect_pass=False)
33+
34+
35+
class TestDeleteInsertWithoutUniqueKeyRaises:
36+
"""delete+insert without unique_key -> validate.sql raises the missing-unique_key error
37+
on the first run (file_format stays delta so only the unique_key branch fires)."""
38+
39+
@pytest.fixture(scope="class")
40+
def project_config_update(self):
41+
return {"models": {"+incremental_strategy": "delete+insert"}}
42+
43+
@pytest.fixture(scope="class")
44+
def models(self):
45+
return {"delete_insert_no_key.sql": fixtures.base_model}
46+
47+
def test_delete_insert_without_unique_key_raises(self, project):
48+
util.run_dbt(["run"], expect_pass=False)
49+
50+
51+
class TestMergeUpdateAndExcludeColumnsRaises(RerunSafeMixin):
52+
"""merge with BOTH merge_update_columns and merge_exclude_columns ->
53+
databricks__get_merge_update_columns (strategies.sql) raises. The raise lives on
54+
the merge build path, so it fires on the second (incremental) run."""
55+
56+
@pytest.fixture(scope="class")
57+
def relations_to_reset(self):
58+
return ("merge_update_exclude",)
59+
60+
@pytest.fixture(scope="class")
61+
def project_config_update(self):
62+
return {
63+
"models": {
64+
"+incremental_strategy": "merge",
65+
"+unique_key": "id",
66+
"+merge_update_columns": ["msg"],
67+
"+merge_exclude_columns": ["msg"],
68+
}
69+
}
70+
71+
@pytest.fixture(scope="class")
72+
def models(self):
73+
return {"merge_update_exclude.sql": fixtures.base_model}
74+
75+
def test_both_merge_column_configs_raise(self, project):
76+
# First run creates the relation (no merge yet) and must succeed.
77+
util.run_dbt(["run"])
78+
# Second run takes the merge path and the mutual-exclusion guard fires.
79+
util.run_dbt(["run"], expect_pass=False)

0 commit comments

Comments
 (0)