Fix(macros): incrementally resolve macro var definitions (#1253)

Author: georgesittas

sqlmesh/core/macros.py

@@ -124,7 +124,9 @@ def send(
             print_exception(e, self.python_env)
             raise MacroEvalError(f"Error trying to eval macro.") from e
 
-    def transform(self, query: exp.Expression) -> exp.Expression | t.List[exp.Expression] | None:
+    def transform(
+        self, expression: exp.Expression
+    ) -> exp.Expression | t.List[exp.Expression] | None:
         changed = False
 
         def _transform_node(node: exp.Expression) -> exp.Expression:
@@ -141,7 +143,7 @@ def _transform_node(node: exp.Expression) -> exp.Expression:
                 return node
             return node
 
-        query = query.transform(_transform_node)
+        expression = expression.transform(_transform_node)
 
         def evaluate_macros(
             node: exp.Expression,
@@ -156,7 +158,7 @@ def evaluate_macros(
                 return self.evaluate(node)
             return node
 
-        transformed = evaluate_macros(query)
+        transformed = evaluate_macros(expression)
 
         if changed:
             # the transformations could have corrupted the ast, turning this into sql and reparsing ensures
@@ -182,7 +184,7 @@ def template(self, text: t.Any, local_variables: t.Dict[str, t.Any]) -> str:
 
     def evaluate(self, node: MacroFunc) -> exp.Expression | t.List[exp.Expression] | None:
         if isinstance(node, MacroDef):
-            self.locals[node.name] = node.expression
+            self.locals[node.name] = self.transform(node.expression)
             return node
 
         if isinstance(node, (MacroSQL, MacroStrReplace)):

tests/core/test_model.py

@@ -55,6 +55,7 @@ def test_load(assert_exp_eq):
         );
 
         @DEF(x, 1);
+        @DEF(y, @x + 1);
         CACHE TABLE x AS SELECT 1;
         ADD JAR 's3://my_jar.jar';
 
@@ -65,6 +66,7 @@ def test_load(assert_exp_eq):
             1::int AS d, -- d
             CAST(2 AS double) AS e, --e
             f::bool, --f
+            @y::int AS g,
         FROM
             db.other_table t1
             LEFT JOIN
@@ -90,13 +92,13 @@ def test_load(assert_exp_eq):
         "d": exp.DataType.build("int"),
         "e": exp.DataType.build("double"),
         "f": exp.DataType.build("boolean"),
+        "g": exp.DataType.build("int"),
     }
     assert model.view_name == "table"
-    assert model.macro_definitions == [
-        d.parse_one("@DEF(x, 1)"),
-    ]
+    assert model.macro_definitions == [d.parse_one("@DEF(x, 1)"), d.parse_one("@DEF(y, @x + 1)")]
     assert list(model.pre_statements) == [
         d.parse_one("@DEF(x, 1)"),
+        d.parse_one("@DEF(y, @x + 1)"),
         d.parse_one("CACHE TABLE x AS SELECT 1"),
         d.parse_one("ADD JAR 's3://my_jar.jar'", dialect="spark"),
     ]
@@ -113,7 +115,8 @@ def test_load(assert_exp_eq):
       TRY_CAST("c" AS BOOLEAN) AS "c",
       TRY_CAST(1 AS INT) AS "d", /* d */
       TRY_CAST(2 AS DOUBLE) AS "e", /* e */
-      TRY_CAST("f" AS BOOLEAN) /* f */ AS "f"
+      TRY_CAST("f" AS BOOLEAN) AS "f", /* f */
+      TRY_CAST(1 + 1 AS INT) AS "g",
     FROM "db"."other_table" AS "t1"
     LEFT JOIN "db"."table" AS "t2"
       ON "t1"."a" = "t2"."a"