fix!: remove most fields from dbt target (#1601)

Author: eakmanrq

sqlmesh/dbt/target.py

@@ -42,6 +42,14 @@
     t.Type[IncrementalUnmanagedKind],
 ]
 
+# We only serialize a subset of fields in order to avoid persisting sensitive information
+SERIALIZABLE_FIELDS = {
+    "type",
+    "name",
+    "database",
+    "schema_",
+}
+
 
 class TargetConfig(abc.ABC, DbtConfig):
     """
@@ -99,7 +107,7 @@ def to_sqlmesh(self) -> ConnectionConfig:
         raise NotImplementedError
 
     def attribute_dict(self) -> AttributeDict:
-        fields = self.dict().copy()
+        fields = self.dict(include=SERIALIZABLE_FIELDS).copy()
         fields["target_name"] = self.name
         return AttributeDict(fields)
 

sqlmesh/migrations/v0031_remove_dbt_target_fields.py

@@ -0,0 +1,64 @@
+"""Remove dbt target fields from snapshots outside of limited list of approved fields"""
+import json
+
+import pandas as pd
+from sqlglot import exp
+
+from sqlmesh.utils.migration import index_text_type
+
+
+def migrate(state_sync):  # type: ignore
+    engine_adapter = state_sync.engine_adapter
+    schema = state_sync.schema
+    snapshots_table = "_snapshots"
+    if schema:
+        snapshots_table = f"{schema}.{snapshots_table}"
+
+    new_snapshots = []
+    found_dbt_target = False
+    for name, identifier, version, snapshot, kind_name in engine_adapter.fetchall(
+        exp.select("name", "identifier", "version", "snapshot", "kind_name").from_(snapshots_table),
+        quote_identifiers=True,
+    ):
+        parsed_snapshot = json.loads(snapshot)
+        node = parsed_snapshot["node"]
+        dbt_target = node.get("jinja_macros", {}).get("global_objs", {}).get("target", {})
+        # Double check that `target_name` exists as a field since we know that all dbt targets have `target_name`
+        # We do this in case someone has a target macro defined that is not related to dbt
+        if dbt_target and dbt_target.get("target_name"):
+            found_dbt_target = True
+            node["jinja_macros"]["global_objs"]["target"] = {
+                "type": dbt_target.get("type", "None"),
+                "name": dbt_target.get("name", "None"),
+                "schema": dbt_target.get("schema", "None"),
+                "database": dbt_target.get("database", "None"),
+                "target_name": dbt_target["target_name"],
+            }
+
+        new_snapshots.append(
+            {
+                "name": name,
+                "identifier": identifier,
+                "version": version,
+                "snapshot": json.dumps(parsed_snapshot),
+                "kind_name": kind_name,
+            }
+        )
+
+    if found_dbt_target:
+        engine_adapter.delete_from(snapshots_table, "TRUE")
+
+        text_type = index_text_type(engine_adapter.dialect)
+
+        engine_adapter.insert_append(
+            snapshots_table,
+            pd.DataFrame(new_snapshots),
+            columns_to_types={
+                "name": exp.DataType.build(text_type),
+                "identifier": exp.DataType.build(text_type),
+                "version": exp.DataType.build(text_type),
+                "snapshot": exp.DataType.build("text"),
+                "kind_name": exp.DataType.build(text_type),
+            },
+            contains_json=True,
+        )

tests/dbt/test_transformation.py

@@ -1,3 +1,4 @@
+import json
 import typing as t
 from pathlib import Path
 
@@ -19,6 +20,7 @@
     SqlModel,
     ViewKind,
 )
+from sqlmesh.core.state_sync.engine_adapter import _snapshot_to_json
 from sqlmesh.dbt.builtin import _relation_info_to_relation
 from sqlmesh.dbt.column import (
     ColumnConfig,
@@ -236,7 +238,9 @@ def test_target_jinja(sushi_test_project: Project):
     assert context.render("{{ target.name }}") == "in_memory"
     assert context.render("{{ target.schema }}") == "sushi"
     assert context.render("{{ target.type }}") == "duckdb"
-    assert context.render("{{ target.profile_name }}") == "sushi"
+    # Path and Profile name are not included in serializable fields
+    assert context.render("{{ target.path }}") == "None"
+    assert context.render("{{ target.profile_name }}") == "None"
 
 
 def test_project_name_jinja(sushi_test_project: Project):
@@ -650,3 +654,15 @@ def test_bigquery_table_properties(sushi_test_project: Project, mocker: MockerFi
     ).table_properties == {
         "partition_expiration_days": exp.convert(7),
     }
+
+
+def test_snapshot_json_payload():
+    sushi_context = Context(paths=["tests/fixtures/dbt/sushi_test"])
+    snapshot_json = json.loads(_snapshot_to_json(sushi_context.snapshots["sushi.top_waiters"]))
+    assert snapshot_json["node"]["jinja_macros"]["global_objs"]["target"] == {
+        "type": "duckdb",
+        "name": "in_memory",
+        "schema": "sushi",
+        "database": "memory",
+        "target_name": "in_memory",
+    }