Feat: dbt namespace macros (#690)

* Implement dbt namespace macros

* PR review fixes

Author: crericha

sqlmesh/dbt/basemodel.py

@@ -17,7 +17,6 @@
 from sqlmesh.core.config.base import UpdateStrategy
 from sqlmesh.core.model import Model
 from sqlmesh.dbt.adapter import ParsetimeAdapter
-from sqlmesh.dbt.builtin import create_builtin_globals
 from sqlmesh.dbt.column import (
     ColumnConfig,
     column_descriptions_to_sqlmesh,
@@ -276,6 +275,8 @@ def _context_for_dependencies(
 
 class ModelSqlRenderer(t.Generic[BMC]):
     def __init__(self, context: DbtContext, config: BMC):
+        from sqlmesh.dbt.builtin import create_builtin_globals
+
         self.context = context
         self.config = config
 

sqlmesh/dbt/builtin.py

@@ -2,8 +2,10 @@
 
 import json
 import os
+import sys
 import typing as t
 from ast import literal_eval
+from pathlib import Path
 
 import agate
 import jinja2
@@ -13,6 +15,8 @@
 
 from sqlmesh.core.engine_adapter import EngineAdapter
 from sqlmesh.dbt.adapter import ParsetimeAdapter, RuntimeAdapter
+from sqlmesh.dbt.common import DbtContext
+from sqlmesh.dbt.package import PackageLoader
 from sqlmesh.utils import AttributeDict, yaml
 from sqlmesh.utils.errors import ConfigError, MacroEvalError
 from sqlmesh.utils.jinja import JinjaMacroRegistry, MacroReturnVal
@@ -246,6 +250,26 @@ def _try_literal_eval(value: str) -> t.Any:
         return value
 
 
+def _dbt_macros_registry() -> JinjaMacroRegistry:
+    registry = JinjaMacroRegistry()
+
+    try:
+        site_packages = next(
+            p for p in sys.path if "site-packages" in p and Path(p, "dbt").exists()
+        )
+    except:
+        return registry
+
+    for project_file in Path(site_packages).glob("dbt/include/*/dbt_project.yml"):
+        if project_file.parent.stem == "starter_project":
+            continue
+        context = DbtContext(project_root=project_file.parent, jinja_macros=JinjaMacroRegistry())
+        package = PackageLoader(context).load()
+        registry.add_macros(package.macros, package="dbt")
+
+    return registry
+
+
 BUILTIN_GLOBALS = {
     "api": Api(),
     "env_var": env_var,
@@ -341,7 +365,13 @@ def create_builtin_globals(
             }
         )
 
-    return {**builtin_globals, **jinja_globals}
+    builtin_globals.update(jinja_globals)
+    if "dbt" not in builtin_globals:
+        builtin_globals["dbt"] = (
+            _dbt_macros_registry().build_environment(**builtin_globals).globals.get("dbt", {})
+        )
+
+    return builtin_globals
 
 
 def create_builtin_filters() -> t.Dict[str, t.Callable]:

sqlmesh/dbt/common.py

@@ -68,7 +68,7 @@ class DbtContext:
     _sources: t.Dict[str, SourceConfig] = field(default_factory=dict)
     _refs: t.Dict[str, str] = field(default_factory=dict)
 
-    _target: t.Optional[TargetConfig] = None
+    _target: TargetConfig = field(default_factory=TargetConfig)
 
     _jinja_environment: t.Optional[Environment] = None
 
@@ -140,8 +140,6 @@ def refs(self) -> t.Dict[str, str]:
 
     @property
     def target(self) -> TargetConfig:
-        if not self._target:
-            raise ConfigError(f"Target not set for {self.project_name}")
         return self._target
 
     @target.setter

sqlmesh/dbt/package.py

@@ -48,9 +48,9 @@ class Package(PydanticModel):
 class PackageLoader:
     """Loader for DBT packages"""
 
-    def __init__(self, context: DbtContext, overrides: ProjectConfig):
+    def __init__(self, context: DbtContext, overrides: t.Optional[ProjectConfig] = None):
         self._context = context.copy()
-        self._overrides = overrides
+        self._overrides = overrides or ProjectConfig()
         self._config_paths: t.Set[Path] = set()
         self.project_config = ProjectConfig()
 

sqlmesh/dbt/target.py

@@ -40,9 +40,9 @@ class TargetConfig(abc.ABC, PydanticModel):
     """
 
     # dbt
-    type: str
+    type: str = ""
     name: str = ""
-    schema_: str = Field(alias="schema")
+    schema_: str = Field(default="", alias="schema")
     threads: int = 1
     profile_name: t.Optional[str] = None
 

sqlmesh/utils/jinja.py

@@ -211,14 +211,15 @@ def build_macro(self, reference: MacroReference, **kwargs: t.Any) -> t.Optional[
         Returns:
             The macro as a Python callable or None if not found.
         """
+        global_vars = self._create_builtin_globals(kwargs)
         if reference.package is not None and reference.name not in self.packages.get(
             reference.package, {}
         ):
-            return None
+            global_package = global_vars.get(reference.package, {})
+            return global_package.get(reference.name)
         if reference.package is None and reference.name not in self.root_macros:
-            return None
+            return global_vars.get(reference.name)
 
-        global_vars = self._create_builtin_globals(kwargs)
         return self._make_callable(reference.name, reference.package, {}, global_vars)
 
     def build_environment(self, **kwargs: t.Any) -> Environment:

tests/dbt/test_transformation.py

@@ -5,13 +5,15 @@
 from dbt.exceptions import CompilationError
 from sqlglot import exp, parse_one
 
+from sqlmesh.core.config.connection import DuckDBConnectionConfig
 from sqlmesh.core.context import Context, ExecutionContext
 from sqlmesh.core.model import (
     IncrementalByTimeRangeKind,
     IncrementalByUniqueKeyKind,
     ModelKind,
     ModelKindName,
 )
+from sqlmesh.dbt.builtin import create_builtin_globals
 from sqlmesh.dbt.column import (
     ColumnConfig,
     column_descriptions_to_sqlmesh,
@@ -426,3 +428,26 @@ def test_zip(sushi_test_project: Project):
     assert context.render("{{ zip_strict([1, 2], ['a', 'b']) }}") == "[(1, 'a'), (2, 'b')]"
     with pytest.raises(TypeError):
         context.render("{{ zip_strict(12, ['a', 'b']) }}")
+
+
+def test_dbt_namespace():
+    context = DbtContext()
+    jinja_globals = create_builtin_globals(
+        jinja_macros=context.jinja_macros,
+        jinja_globals={},
+        engine_adapter=DuckDBConnectionConfig().create_engine_adapter(),
+    )
+    jinja_env = context.jinja_macros.build_environment(**jinja_globals)
+
+    assert (
+        jinja_env.from_string("{{ dbt.replace('original sentence', 'original', 'updated') }}")
+        .render()
+        .strip()
+        == """replace(
+        original sentence,
+        original,
+        updated
+    )"""
+    )
+
+    assert jinja_env.from_string("{{ dbt.hash('col')}}").render() == "md5(cast(col as TEXT))"