Feat: Enable previews by default for native projects and dbt projects if the target engine suports cloning (#3771)

Author: izeigerman

docs/reference/configuration.md

@@ -61,15 +61,15 @@ Global variable values may be any of the data types in the table below or lists
 
 Configuration for the `sqlmesh plan` command.
 
-| Option                    | Description                                                                                                                                                                                                                                             |         Type         | Required |
-| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------: | :------: |
-| `auto_categorize_changes` | Indicates whether SQLMesh should attempt to automatically [categorize](../concepts/plans.md#change-categories) model changes during plan creation per each model source type ([additional details](../guides/configuration.md#auto-categorize-changes)) | dict[string, string] |    N     |
-| `include_unmodified`      | Indicates whether to create views for all models in the target development environment or only for modified ones (Default: False)                                                                                                                       |       boolean        |    N     |
-| `auto_apply`              | Indicates whether to automatically apply a new plan after creation (Default: False)                                                                                                                                                                     |       boolean        |    N     |
-| `forward_only`            | Indicates whether the plan should be [forward-only](../concepts/plans.md#forward-only-plans) (Default: False)                                                                                                                                           |       boolean        |    N     |
-| `enable_preview`          | Indicates whether to enable [data preview](../concepts/plans.md#data-preview) for forward-only models when targeting a development environment (Default: False)                                                                                         |       boolean        |    N     |
-| `no_diff`                 | Don't show diffs for changed models (Default: False)                                                                                                                                                                                                    |       boolean        |    N     |
-| `no_prompts`              | Disables interactive prompts in CLI (Default: True)                                                                                                                                                                                                    |       boolean        |    N     |
+| Option                    | Description                                                                                                                                                                                                                                             | Type                 | Required |
+|---------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:--------------------:|:--------:|
+| `auto_categorize_changes` | Indicates whether SQLMesh should attempt to automatically [categorize](../concepts/plans.md#change-categories) model changes during plan creation per each model source type ([additional details](../guides/configuration.md#auto-categorize-changes)) | dict[string, string] | N        |
+| `include_unmodified`      | Indicates whether to create views for all models in the target development environment or only for modified ones (Default: False)                                                                                                                       | boolean              | N        |
+| `auto_apply`              | Indicates whether to automatically apply a new plan after creation (Default: False)                                                                                                                                                                     | boolean              | N        |
+| `forward_only`            | Indicates whether the plan should be [forward-only](../concepts/plans.md#forward-only-plans) (Default: False)                                                                                                                                           | boolean              | N        |
+| `enable_preview`          | Indicates whether to enable [data preview](../concepts/plans.md#data-preview) for forward-only models when targeting a development environment (Default: True, except for dbt projects where the target engine does not support cloning)                | Boolean              | N        |
+| `no_diff`                 | Don't show diffs for changed models (Default: False)                                                                                                                                                                                                    | boolean              | N        |
+| `no_prompts`              | Disables interactive prompts in CLI (Default: True)                                                                                                                                                                                                     | boolean              | N        |
 
 ## Run
 

sqlmesh/core/config/plan.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import typing as t
+
 from sqlmesh.core.config.base import BaseConfig
 from sqlmesh.core.config.categorizer import CategorizerConfig
 
@@ -23,7 +25,7 @@ class PlanConfig(BaseConfig):
     forward_only: bool = False
     auto_categorize_changes: CategorizerConfig = CategorizerConfig()
     include_unmodified: bool = False
-    enable_preview: bool = False
+    enable_preview: t.Optional[bool] = None
     no_diff: bool = False
     no_prompts: bool = True
     auto_apply: bool = False

sqlmesh/core/context.py

@@ -596,13 +596,8 @@ def load(self, update_schemas: bool = True) -> GenericContext[C]:
             self.default_dialect or ""
         }
 
-        project_types = {
-            c.DBT if loader.config.loader.__name__.lower().startswith(c.DBT) else c.NATIVE
-            for loader in self._loaders
-        }
-
         analytics.collector.on_project_loaded(
-            project_type=c.HYBRID if len(project_types) > 1 else first(project_types),
+            project_type=self._project_type,
             models_count=len(self._models),
             audits_count=len(self._audits),
             standalone_audits_count=len(self._standalone_audits),
@@ -1390,7 +1385,7 @@ def plan_builder(
             default_start=default_start,
             default_end=default_end,
             enable_preview=(
-                enable_preview if enable_preview is not None else self.config.plan.enable_preview
+                enable_preview if enable_preview is not None else self._plan_preview_enabled
             ),
             end_bounded=not run,
             ensure_finalized_snapshots=self.config.plan.use_finalized_state,
@@ -2279,6 +2274,23 @@ def _select_models_for_run(
             result = set(dag.subdag(*result))
         return result
 
+    @cached_property
+    def _project_type(self) -> str:
+        project_types = {
+            c.DBT if loader.__class__.__name__.lower().startswith(c.DBT) else c.NATIVE
+            for loader in self._loaders
+        }
+        return c.HYBRID if len(project_types) > 1 else first(project_types)
+
+    @property
+    def _plan_preview_enabled(self) -> bool:
+        if self.config.plan.enable_preview is not None:
+            return self.config.plan.enable_preview
+        # It is dangerous to enable preview by default for dbt projects that rely on engines that don’t support cloning.
+        # Enabling previews in such cases can result in unintended full refreshes because dbt incremental models rely on
+        # the maximum timestamp value in the target table.
+        return self._project_type == c.NATIVE or self.engine_adapter.SUPPORTS_CLONING
+
 
 class Context(GenericContext[Config]):
     CONFIG_TYPE = Config

sqlmesh/core/model/kind.py

@@ -120,7 +120,7 @@ def is_symbolic(self) -> bool:
 
     @property
     def is_materialized(self) -> bool:
-        return not (self.is_symbolic or self.is_view)
+        return self.model_kind_name is not None and not (self.is_symbolic or self.is_view)
 
     @property
     def only_execution_time(self) -> bool:

tests/core/test_context.py

@@ -1265,3 +1265,11 @@ def test_rendered_diff():
  )
 -DROP VIEW "test"
 +DROP VIEW IF EXISTS "test"''' in plan.context_diff.text_diff('"test"')
+
+
+def test_plan_enable_preview_default(sushi_context: Context, sushi_dbt_context: Context):
+    assert sushi_context._plan_preview_enabled
+    assert not sushi_dbt_context._plan_preview_enabled
+
+    sushi_dbt_context.engine_adapter.SUPPORTS_CLONING = True
+    assert sushi_dbt_context._plan_preview_enabled

tests/core/test_integration.py

@@ -232,7 +232,7 @@ def test_forward_only_model_regular_plan(init_and_plan_context: t.Callable):
     snapshot = context.get_snapshot(model, raise_if_missing=True)
     top_waiters_snapshot = context.get_snapshot("sushi.top_waiters", raise_if_missing=True)
 
-    plan = context.plan_builder("dev", skip_tests=True).build()
+    plan = context.plan_builder("dev", skip_tests=True, enable_preview=False).build()
     assert len(plan.new_snapshots) == 2
     assert (
         plan.context_diff.snapshots[snapshot.snapshot_id].change_category
@@ -253,7 +253,9 @@ def test_forward_only_model_regular_plan(init_and_plan_context: t.Callable):
     assert not dev_df["event_date"].tolist()
 
     # Run a restatement plan to preview changes
-    plan_builder = context.plan_builder("dev", skip_tests=True, restate_models=[model_name])
+    plan_builder = context.plan_builder(
+        "dev", skip_tests=True, restate_models=[model_name], enable_preview=False
+    )
     plan_builder.set_start("2023-01-06")
     assert plan_builder.build().missing_intervals == [
         SnapshotIntervals(
@@ -397,7 +399,7 @@ def test_forward_only_model_restate_full_history_in_dev(init_and_plan_context: t
     assert model.kind.full_history_restatement_only
     context.upsert_model(model)
 
-    context.plan("prod", skip_tests=True, auto_apply=True)
+    context.plan("prod", skip_tests=True, auto_apply=True, enable_preview=False)
 
     model_kwargs = {
         **model.dict(),
@@ -407,7 +409,7 @@ def test_forward_only_model_restate_full_history_in_dev(init_and_plan_context: t
     context.upsert_model(SqlModel.parse_obj(model_kwargs))
 
     # Apply the model change in dev
-    plan = context.plan_builder("dev", skip_tests=True).build()
+    plan = context.plan_builder("dev", skip_tests=True, enable_preview=False).build()
     assert not plan.missing_intervals
     context.apply(plan)
 
@@ -425,7 +427,7 @@ def test_forward_only_model_restate_full_history_in_dev(init_and_plan_context: t
     assert df["cnt"][0] == 1
 
     # Apply a restatement plan in dev
-    plan = context.plan("dev", restate_models=[model.name], auto_apply=True)
+    plan = context.plan("dev", restate_models=[model.name], auto_apply=True, enable_preview=False)
     assert len(plan.missing_intervals) == 1
 
     # Check that the dummy value is not present
@@ -833,7 +835,7 @@ def test_forward_only_parent_created_in_dev_child_created_in_prod(
     )
     top_waiters_snapshot = context.get_snapshot("sushi.top_waiters", raise_if_missing=True)
 
-    plan = context.plan_builder("dev", skip_tests=True).build()
+    plan = context.plan_builder("dev", skip_tests=True, enable_preview=False).build()
     assert len(plan.new_snapshots) == 2
     assert (
         plan.context_diff.snapshots[waiter_revenue_by_day_snapshot.snapshot_id].change_category
@@ -855,7 +857,7 @@ def test_forward_only_parent_created_in_dev_child_created_in_prod(
 
     top_waiters_snapshot = context.get_snapshot("sushi.top_waiters", raise_if_missing=True)
 
-    plan = context.plan_builder("prod", skip_tests=True).build()
+    plan = context.plan_builder("prod", skip_tests=True, enable_preview=False).build()
     assert len(plan.new_snapshots) == 1
     assert (
         plan.context_diff.snapshots[top_waiters_snapshot.snapshot_id].change_category
@@ -869,7 +871,7 @@ def test_forward_only_parent_created_in_dev_child_created_in_prod(
 def test_new_forward_only_model(init_and_plan_context: t.Callable):
     context, _ = init_and_plan_context("examples/sushi")
 
-    context.plan("dev", skip_tests=True, no_prompts=True, auto_apply=True)
+    context.plan("dev", skip_tests=True, no_prompts=True, auto_apply=True, enable_preview=False)
 
     snapshot = context.get_snapshot("sushi.marketing")
 
@@ -1156,7 +1158,7 @@ def test_indirect_non_breaking_change_after_forward_only_in_dev(init_and_plan_co
     context.upsert_model(model)
     snapshot = context.get_snapshot(model, raise_if_missing=True)
 
-    plan = context.plan_builder("dev", skip_tests=True).build()
+    plan = context.plan_builder("dev", skip_tests=True, enable_preview=False).build()
     assert (
         plan.context_diff.snapshots[snapshot.snapshot_id].change_category
         == SnapshotChangeCategory.FORWARD_ONLY
@@ -1169,7 +1171,7 @@ def test_indirect_non_breaking_change_after_forward_only_in_dev(init_and_plan_co
     context.upsert_model(add_projection_to_model(t.cast(SqlModel, model)))
     top_waiters_snapshot = context.get_snapshot("sushi.top_waiters", raise_if_missing=True)
 
-    plan = context.plan_builder("dev", skip_tests=True).build()
+    plan = context.plan_builder("dev", skip_tests=True, enable_preview=False).build()
     assert len(plan.new_snapshots) == 1
     assert (
         plan.context_diff.snapshots[top_waiters_snapshot.snapshot_id].change_category
@@ -1202,7 +1204,7 @@ def test_indirect_non_breaking_change_after_forward_only_in_dev(init_and_plan_co
     )
     top_waiters_snapshot = context.get_snapshot("sushi.top_waiters", raise_if_missing=True)
 
-    plan = context.plan_builder("dev", skip_tests=True).build()
+    plan = context.plan_builder("dev", skip_tests=True, enable_preview=False).build()
     assert len(plan.new_snapshots) == 2
     assert (
         plan.context_diff.snapshots[waiter_revenue_by_day_snapshot.snapshot_id].change_category
@@ -1233,7 +1235,7 @@ def test_indirect_non_breaking_change_after_forward_only_in_dev(init_and_plan_co
     assert not context.plan_builder("dev", skip_tests=True).build().requires_backfill
 
     # Deploy everything to prod.
-    plan = context.plan_builder("prod", skip_tests=True).build()
+    plan = context.plan_builder("prod", skip_tests=True, enable_preview=False).build()
     assert plan.start == to_timestamp("2023-01-01")
     assert plan.missing_intervals == [
         SnapshotIntervals(
@@ -1263,7 +1265,11 @@ def test_indirect_non_breaking_change_after_forward_only_in_dev(init_and_plan_co
     ]
 
     context.apply(plan)
-    assert not context.plan_builder("prod", skip_tests=True).build().requires_backfill
+    assert (
+        not context.plan_builder("prod", skip_tests=True, enable_preview=False)
+        .build()
+        .requires_backfill
+    )
 
 
 @time_machine.travel("2023-01-08 15:00:00 UTC")
@@ -1285,7 +1291,7 @@ def test_forward_only_precedence_over_indirect_non_breaking(init_and_plan_contex
     non_breaking_snapshot = context.get_snapshot(non_breaking_model, raise_if_missing=True)
     top_waiter_snapshot = context.get_snapshot("sushi.top_waiters", raise_if_missing=True)
 
-    plan = context.plan_builder("dev", skip_tests=True).build()
+    plan = context.plan_builder("dev", skip_tests=True, enable_preview=False).build()
     assert (
         plan.context_diff.snapshots[forward_only_snapshot.snapshot_id].change_category
         == SnapshotChangeCategory.FORWARD_ONLY
@@ -1315,7 +1321,11 @@ def test_forward_only_precedence_over_indirect_non_breaking(init_and_plan_contex
     ]
 
     context.apply(plan)
-    assert not context.plan_builder("dev", skip_tests=True).build().requires_backfill
+    assert (
+        not context.plan_builder("dev", skip_tests=True, enable_preview=False)
+        .build()
+        .requires_backfill
+    )
 
     # Deploy everything to prod.
     plan = context.plan_builder("prod", skip_tests=True).build()
@@ -1336,7 +1346,11 @@ def test_forward_only_precedence_over_indirect_non_breaking(init_and_plan_contex
     ]
 
     context.apply(plan)
-    assert not context.plan_builder("prod", skip_tests=True).build().requires_backfill
+    assert (
+        not context.plan_builder("prod", skip_tests=True, enable_preview=False)
+        .build()
+        .requires_backfill
+    )
 
 
 @time_machine.travel("2023-01-08 15:00:00 UTC")
@@ -1579,12 +1593,6 @@ def test_select_models_for_backfill(init_and_plan_context: t.Callable):
     context, _ = init_and_plan_context("examples/sushi")
 
     expected_intervals = [
-        (to_timestamp("2023-01-01"), to_timestamp("2023-01-02")),
-        (to_timestamp("2023-01-02"), to_timestamp("2023-01-03")),
-        (to_timestamp("2023-01-03"), to_timestamp("2023-01-04")),
-        (to_timestamp("2023-01-04"), to_timestamp("2023-01-05")),
-        (to_timestamp("2023-01-05"), to_timestamp("2023-01-06")),
-        (to_timestamp("2023-01-06"), to_timestamp("2023-01-07")),
         (to_timestamp("2023-01-07"), to_timestamp("2023-01-08")),
     ]
 
@@ -1620,7 +1628,7 @@ def test_select_models_for_backfill(init_and_plan_context: t.Callable):
     dev_df = context.engine_adapter.fetchdf(
         "SELECT DISTINCT event_date FROM sushi__dev.waiter_revenue_by_day ORDER BY event_date"
     )
-    assert len(dev_df) == 7
+    assert len(dev_df) == 1
 
     schema_objects = context.engine_adapter.get_data_objects("sushi__dev")
     assert {o.name for o in schema_objects} == {
@@ -1854,7 +1862,7 @@ def test_indirect_non_breaking_view_model_non_representative_snapshot(
     full_downstream_model_2_snapshot_id = context.get_snapshot(
         view_downstream_model_name
     ).snapshot_id
-    dev_plan = context.plan("dev", auto_apply=True, no_prompts=True)
+    dev_plan = context.plan("dev", auto_apply=True, no_prompts=True, enable_preview=False)
     assert (
         dev_plan.snapshots[forward_only_model_snapshot_id].change_category
         == SnapshotChangeCategory.FORWARD_ONLY
@@ -1887,7 +1895,11 @@ def test_indirect_non_breaking_view_model_non_representative_snapshot(
         view_downstream_model_name
     ).snapshot_id
     dev_plan = context.plan(
-        "dev", categorizer_config=CategorizerConfig.all_full(), auto_apply=True, no_prompts=True
+        "dev",
+        categorizer_config=CategorizerConfig.all_full(),
+        auto_apply=True,
+        no_prompts=True,
+        enable_preview=False,
     )
     assert (
         dev_plan.snapshots[full_downstream_model_snapshot_id].change_category
@@ -1917,7 +1929,11 @@ def test_indirect_non_breaking_view_model_non_representative_snapshot(
         view_downstream_model_name
     ).snapshot_id
     dev_plan = context.plan(
-        "dev", categorizer_config=CategorizerConfig.all_full(), auto_apply=True, no_prompts=True
+        "dev",
+        categorizer_config=CategorizerConfig.all_full(),
+        auto_apply=True,
+        no_prompts=True,
+        enable_preview=False,
     )
     assert (
         dev_plan.snapshots[full_downstream_model_snapshot_id].change_category