feat: Jinja/dbt template preprocessing for SQL analysis tools

packages/altimate-code/src/bridge/protocol.ts

@@ -951,6 +951,25 @@ export interface LocalTestResult {
   error?: string
 }
 
+// --- Jinja Preprocessing ---
+
+export interface SqlPreprocessJinjaParams {
+  sql: string
+}
+
+export interface SqlPreprocessJinjaResult {
+  success: boolean
+  preprocessed_sql: string
+  original_sql: string
+  was_preprocessed: boolean
+  refs_found: string[]
+  sources_found: string[]
+  variables_found: string[]
+  macros_removed: string[]
+  warnings: string[]
+  error?: string
+}
+
 // --- Method registry ---
 
 export const BridgeMethods = {
@@ -986,6 +1005,7 @@ export const BridgeMethods = {
   "schema.detect_pii": {} as { params: PiiDetectParams; result: PiiDetectResult },
   "schema.tags": {} as { params: TagsGetParams; result: TagsGetResult },
   "schema.tags_list": {} as { params: TagsListParams; result: TagsListResult },
+  "sql.preprocess_jinja": {} as { params: SqlPreprocessJinjaParams; result: SqlPreprocessJinjaResult },
   "sql.diff": {} as { params: SqlDiffParams; result: SqlDiffResult },
   "sql.rewrite": {} as { params: SqlRewriteParams; result: SqlRewriteResult },
   "sql.schema_diff": {} as { params: SchemaDiffParams; result: SchemaDiffResult },

packages/altimate-code/src/tool/registry.ts

@@ -56,6 +56,7 @@ import { FinopsRoleGrantsTool, FinopsRoleHierarchyTool, FinopsUserRolesTool } fr
 import { SchemaDetectPiiTool } from "./schema-detect-pii"
 import { SchemaTagsTool, SchemaTagsListTool } from "./schema-tags"
 import { SqlRewriteTool } from "./sql-rewrite"
+import { SqlPreprocessJinjaTool } from "./sql-preprocess-jinja"
 
 import { SchemaDiffTool } from "./schema-diff"
 import { AltimateCoreValidateTool } from "./altimate-core-validate"
@@ -219,6 +220,7 @@ export namespace ToolRegistry {
       SchemaTagsTool,
       SchemaTagsListTool,
       SqlRewriteTool,
+      SqlPreprocessJinjaTool,
       SchemaDiffTool,
       AltimateCoreValidateTool,
       AltimateCoreLintTool,

packages/altimate-code/src/tool/sql-preprocess-jinja.ts

@@ -0,0 +1,103 @@
+import z from "zod"
+import { Tool } from "./tool"
+import { Bridge } from "../bridge/client"
+import type { SqlPreprocessJinjaResult } from "../bridge/protocol"
+
+export const SqlPreprocessJinjaTool = Tool.define("sql_preprocess_jinja", {
+  description:
+    "Preprocess Jinja/dbt template syntax in SQL before analysis. Stubs common dbt macros like {{ ref() }}, {{ source() }}, {{ config() }}, {{ var() }}, {{ this }}, and Jinja block tags ({% if %}, {% for %}) into plain SQL that downstream tools can parse. Use this when SQL analysis tools fail on dbt-templated SQL.",
+  parameters: z.object({
+    sql: z.string().describe("SQL with Jinja/dbt template syntax to preprocess"),
+  }),
+  async execute(args, ctx) {
+    try {
+      const result = await Bridge.call("sql.preprocess_jinja", {
+        sql: args.sql,
+      })
+
+      if (!result.was_preprocessed) {
+        return {
+          title: "Preprocess Jinja: no templates found",
+          metadata: {
+            success: true as boolean,
+            was_preprocessed: false as boolean,
+            refs: [] as string[],
+            sources: [] as string[],
+            variables: [] as string[],
+          },
+          output: "No Jinja templates detected in the SQL. The input is already plain SQL.",
+        }
+      }
+
+      return {
+        title: `Preprocess Jinja: ${formatSummary(result)}`,
+        metadata: {
+          success: result.success,
+          was_preprocessed: result.was_preprocessed,
+          refs: result.refs_found,
+          sources: result.sources_found,
+          variables: result.variables_found,
+        },
+        output: formatResult(result),
+      }
+    } catch (e) {
+      const msg = e instanceof Error ? e.message : String(e)
+      return {
+        title: "Preprocess Jinja: ERROR",
+        metadata: {
+          success: false as boolean,
+          was_preprocessed: false as boolean,
+          refs: [] as string[],
+          sources: [] as string[],
+          variables: [] as string[],
+        },
+        output: `Failed to preprocess Jinja: ${msg}\n\nEnsure the Python bridge is running and altimate-engine is installed.`,
+      }
+    }
+  },
+})
+
+function formatSummary(result: SqlPreprocessJinjaResult): string {
+  const parts: string[] = []
+  if (result.refs_found.length > 0) parts.push(`${result.refs_found.length} ref(s)`)
+  if (result.sources_found.length > 0) parts.push(`${result.sources_found.length} source(s)`)
+  if (result.variables_found.length > 0) parts.push(`${result.variables_found.length} var(s)`)
+  return parts.length > 0 ? parts.join(", ") : "templates removed"
+}
+
+function formatResult(result: SqlPreprocessJinjaResult): string {
+  const lines: string[] = []
+
+  lines.push("=== Preprocessed SQL ===")
+  lines.push(result.preprocessed_sql)
+  lines.push("")
+
+  if (result.refs_found.length > 0) {
+    lines.push(`Models referenced (ref): ${result.refs_found.join(", ")}`)
+  }
+  if (result.sources_found.length > 0) {
+    lines.push(`Sources referenced: ${result.sources_found.join(", ")}`)
+  }
+  if (result.variables_found.length > 0) {
+    lines.push(`Variables used (var): ${result.variables_found.join(", ")}`)
+  }
+  if (result.macros_removed.length > 0) {
+    lines.push(`Macros removed: ${result.macros_removed.join(", ")}`)
+  }
+
+  if (result.warnings.length > 0) {
+    lines.push("")
+    lines.push("=== Warnings ===")
+    for (const w of result.warnings) {
+      lines.push(`  ! ${w}`)
+    }
+  }
+
+  lines.push("")
+  lines.push(
+    "Note: Jinja templates were stubbed with placeholder values. " +
+      "Analysis results on this SQL are approximate.",
+  )
+
+  return lines.join("\n")
+}

packages/altimate-engine/src/altimate_engine/models.py

@@ -987,6 +987,26 @@ class AltimateCoreIsSafeParams(BaseModel):
     sql: str
 
 
+# --- Jinja Preprocessing ---
+
+
+class SqlPreprocessJinjaParams(BaseModel):
+    sql: str
+
+
+class SqlPreprocessJinjaResult(BaseModel):
+    success: bool = True
+    preprocessed_sql: str
+    original_sql: str
+    was_preprocessed: bool
+    refs_found: list[str] = Field(default_factory=list)
+    sources_found: list[str] = Field(default_factory=list)
+    variables_found: list[str] = Field(default_factory=list)
+    macros_removed: list[str] = Field(default_factory=list)
+    warnings: list[str] = Field(default_factory=list)
+    error: str = Field(default=None)  # Uses same pattern as rest of file
+
+
 # --- JSON-RPC ---
 
 

packages/altimate-engine/src/altimate_engine/server.py

@@ -170,7 +170,13 @@
 from altimate_engine.dbt.profiles import discover_dbt_connections
 from altimate_engine.local.schema_sync import sync_schema
 from altimate_engine.local.test_local import test_sql_local
+from altimate_engine.sql.jinja_preprocessor import (
+    contains_jinja,
+    preprocess_jinja,
+)
 from altimate_engine.models import (
+    SqlPreprocessJinjaParams,
+    SqlPreprocessJinjaResult,
     AltimateCoreFixParams,
     AltimateCorePolicyParams,
     AltimateCoreSemanticsParams,
@@ -298,13 +304,48 @@ def dispatch(request: JsonRpcRequest) -> JsonRpcResponse:
     params = request.params or {}
 
     try:
-        if method == "sql.execute":
+        if method == "sql.preprocess_jinja":
+            pp_params = SqlPreprocessJinjaParams(**params)
+            pp_result = preprocess_jinja(pp_params.sql)
+            result = SqlPreprocessJinjaResult(
+                success=True,
+                preprocessed_sql=pp_result.preprocessed_sql,
+                original_sql=pp_result.original_sql,
+                was_preprocessed=pp_result.was_preprocessed,
+                refs_found=pp_result.refs_found,
+                sources_found=pp_result.sources_found,
+                variables_found=pp_result.variables_found,
+                macros_removed=pp_result.macros_removed,
+                warnings=pp_result.warnings,
+            )
+        elif method == "sql.execute":
             result = execute_sql(SqlExecuteParams(**params))
         elif method == "schema.inspect":
             result = inspect_schema(SchemaInspectParams(**params))
         elif method == "sql.analyze":
             params_obj = SqlAnalyzeParams(**params)
-            statements = _split_sql_statements(params_obj.sql)
+
+            # Auto-preprocess Jinja if present
+            jinja_note = ""
+            sql_to_analyze = params_obj.sql
+            if contains_jinja(sql_to_analyze):
+                pp = preprocess_jinja(sql_to_analyze)
+                if pp.was_preprocessed:
+                    sql_to_analyze = pp.preprocessed_sql
+                    parts = []
+                    if pp.refs_found:
+                        parts.append(f"refs: {', '.join(pp.refs_found)}")
+                    if pp.sources_found:
+                        parts.append(f"sources: {', '.join(pp.sources_found)}")
+                    if pp.variables_found:
+                        parts.append(f"vars: {', '.join(pp.variables_found)}")
+                    detail = f" ({'; '.join(parts)})" if parts else ""
+                    jinja_note = (
+                        f"Jinja templates were preprocessed before analysis{detail}. "
+                        "Results are based on the rendered SQL and may be approximate."
+                    )
+
+            statements = _split_sql_statements(sql_to_analyze)
             issues = []
             any_error = None
 
@@ -360,37 +401,65 @@ def dispatch(request: JsonRpcRequest) -> JsonRpcResponse:
                         )
                     )
 
+            confidence_factors = []
+            if any_error is not None:
+                confidence_factors.append(f"Parse failed on one statement: {any_error}")
+            if jinja_note:
+                confidence_factors.append(jinja_note)
+
             result = SqlAnalyzeResult(
                 success=any_error is None,
                 issues=issues,
                 issue_count=len(issues),
                 confidence=_compute_overall_confidence(issues),
-                confidence_factors=[]
-                if any_error is None
-                else [f"Parse failed on one statement: {any_error}"],
+                confidence_factors=confidence_factors,
                 error=any_error,
             )
         elif method == "sql.translate":
             params_obj = SqlTranslateParams(**params)
+
+            # Auto-preprocess Jinja if present
+            sql_to_translate = params_obj.sql
+            jinja_warnings: list[str] = []
+            if contains_jinja(sql_to_translate):
+                pp = preprocess_jinja(sql_to_translate)
+                if pp.was_preprocessed:
+                    sql_to_translate = pp.preprocessed_sql
+                    jinja_warnings.append(
+                        "Jinja templates were preprocessed before translation. "
+                        "Review the translated SQL and re-apply Jinja syntax as needed."
+                    )
+
             raw = guard_transpile(
-                params_obj.sql, params_obj.source_dialect, params_obj.target_dialect
+                sql_to_translate, params_obj.source_dialect, params_obj.target_dialect
             )
+            all_warnings = jinja_warnings + raw.get("warnings", [])
             result = SqlTranslateResult(
                 success=raw.get("success", True),
                 translated_sql=raw.get("sql", raw.get("translated_sql")),
                 source_dialect=params_obj.source_dialect,
                 target_dialect=params_obj.target_dialect,
-                warnings=raw.get("warnings", []),
+                warnings=all_warnings,
                 error=raw.get("error"),
             )
         elif method == "sql.optimize":
             params_obj = SqlOptimizeParams(**params)
+
+            # Auto-preprocess Jinja if present
+            sql_to_optimize = params_obj.sql
+            jinja_preprocessed = False
+            if contains_jinja(sql_to_optimize):
+                pp = preprocess_jinja(sql_to_optimize)
+                if pp.was_preprocessed:
+                    sql_to_optimize = pp.preprocessed_sql
+                    jinja_preprocessed = True
+
             # Rewrite for optimization
             rw = guard_rewrite_sql(
-                params_obj.sql, schema_context=params_obj.schema_context
+                sql_to_optimize, schema_context=params_obj.schema_context
             )
             # Lint for remaining issues
-            lint = guard_lint(params_obj.sql, schema_context=params_obj.schema_context)
+            lint = guard_lint(sql_to_optimize, schema_context=params_obj.schema_context)
 
             suggestions = []
             for r in rw.get("rewrites", []):
@@ -424,12 +493,17 @@ def dispatch(request: JsonRpcRequest) -> JsonRpcResponse:
                     }
                 )
 
+            opt_confidence = "high"
+            if jinja_preprocessed:
+                opt_confidence = "medium"
+
             result = SqlOptimizeResult(
                 success=True,
                 original_sql=params_obj.sql,
-                optimized_sql=rw.get("rewritten_sql", params_obj.sql),
+                optimized_sql=rw.get("rewritten_sql", sql_to_optimize),
                 suggestions=suggestions,
                 anti_patterns=anti_patterns,
+                confidence=opt_confidence,
                 error=rw.get("error"),
             )
         elif method == "lineage.check":
@@ -492,12 +566,28 @@ def dispatch(request: JsonRpcRequest) -> JsonRpcResponse:
 
         elif method == "sql.format":
             fmt_params = SqlFormatParams(**params)
-            raw = guard_format_sql(fmt_params.sql, fmt_params.dialect)
+
+            # Auto-preprocess Jinja if present
+            sql_to_format = fmt_params.sql
+            jinja_fmt_note = None
+            if contains_jinja(sql_to_format):
+                pp = preprocess_jinja(sql_to_format)
+                if pp.was_preprocessed:
+                    sql_to_format = pp.preprocessed_sql
+                    jinja_fmt_note = (
+                        "Note: Jinja templates were removed before formatting. "
+                        "The formatted output contains plain SQL only."
+                    )
+
+            raw = guard_format_sql(sql_to_format, fmt_params.dialect)
+            fmt_error = raw.get("error")
+            if jinja_fmt_note and not fmt_error:
+                fmt_error = jinja_fmt_note
             result = SqlFormatResult(
                 success=raw.get("success", True),
                 formatted_sql=raw.get("formatted_sql", raw.get("sql")),
                 statement_count=raw.get("statement_count", 1),
-                error=raw.get("error"),
+                error=fmt_error,
             )
         elif method == "sql.explain":
             result = explain_sql(SqlExplainParams(**params))