fix: improve Jinja preprocessing robustness

- Use __jinja_expr__ placeholder for adapter.dispatch/return/log instead
  of empty string to avoid invalid SQL in expression positions
- Add Jinja auto-preprocessing to lineage.check and sql.explain
- Improve sql.optimize fallback: preserve original SQL when only Jinja
  preprocessing occurred (no actual rewrites)
- Simplify sql.format: drop the SQL comment approach, just format the
  preprocessed SQL cleanly
- Update tests to verify __jinja_expr__ placeholder behavior

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: anandgupta42

packages/altimate-engine/src/altimate_engine/server.py

@@ -472,7 +472,12 @@ def dispatch(request: JsonRpcRequest) -> JsonRpcResponse:
                     )
                 )
 
-            optimized_sql = rw.get("rewritten_sql", params_obj.sql)
+            # When Jinja was preprocessed and no rewrites occurred, return
+            # the original SQL to avoid silently dropping template syntax.
+            if jinja_preprocessed and not suggestions:
+                optimized_sql = params_obj.sql
+            else:
+                optimized_sql = rw.get("rewritten_sql", params_obj.sql)
             if not suggestions and optimized_sql.strip() != params_obj.sql.strip():
                 suggestions.append(
                     SqlOptimizeSuggestion(
@@ -500,16 +505,21 @@ def dispatch(request: JsonRpcRequest) -> JsonRpcResponse:
             result = SqlOptimizeResult(
                 success=True,
                 original_sql=params_obj.sql,
-                optimized_sql=rw.get("rewritten_sql", params_obj.sql),
+                optimized_sql=optimized_sql,
                 suggestions=suggestions,
                 anti_patterns=anti_patterns,
                 confidence=opt_confidence,
                 error=rw.get("error"),
             )
         elif method == "lineage.check":
             p = LineageCheckParams(**params)
+            sql_for_lineage = p.sql
+            if contains_jinja(sql_for_lineage):
+                pp = preprocess_jinja(sql_for_lineage)
+                if pp.was_preprocessed:
+                    sql_for_lineage = pp.preprocessed_sql
             raw = guard_column_lineage(
-                p.sql,
+                sql_for_lineage,
                 dialect=p.dialect or "",
                 schema_context=_schema_context_to_dict(p.schema_context)
                 if p.schema_context
@@ -569,28 +579,33 @@ def dispatch(request: JsonRpcRequest) -> JsonRpcResponse:
 
             # Auto-preprocess Jinja if present
             sql_to_format = fmt_params.sql
-            jinja_fmt_note = None
             if contains_jinja(sql_to_format):
                 pp = preprocess_jinja(sql_to_format)
                 if pp.was_preprocessed:
                     sql_to_format = pp.preprocessed_sql
-                    jinja_fmt_note = (
-                        "Note: Jinja templates were removed before formatting. "
-                        "The formatted output contains plain SQL only."
-                    )
 
             raw = guard_format_sql(sql_to_format, fmt_params.dialect)
             formatted_sql = raw.get("formatted_sql", raw.get("sql"))
-            if jinja_fmt_note and formatted_sql:
-                formatted_sql = formatted_sql.rstrip() + "\n\n-- " + jinja_fmt_note + "\n"
             result = SqlFormatResult(
                 success=raw.get("success", True),
                 formatted_sql=formatted_sql,
                 statement_count=raw.get("statement_count", 1),
-                error=raw.get("error"),  # Only propagate real errors, not Jinja notes
+                error=raw.get("error"),
             )
         elif method == "sql.explain":
-            result = explain_sql(SqlExplainParams(**params))
+            explain_params = SqlExplainParams(**params)
+            sql_to_explain = explain_params.sql
+            if contains_jinja(sql_to_explain):
+                pp = preprocess_jinja(sql_to_explain)
+                if pp.was_preprocessed:
+                    sql_to_explain = pp.preprocessed_sql
+            result = explain_sql(
+                SqlExplainParams(
+                    sql=sql_to_explain,
+                    warehouse=explain_params.warehouse,
+                    analyze=explain_params.analyze,
+                )
+            )
         elif method == "sql.fix":
             fix_params = SqlFixParams(**params)
             guard_result = guard_fix_sql(fix_params.sql)

packages/altimate-engine/src/altimate_engine/sql/jinja_preprocessor.py

@@ -7,7 +7,8 @@
   - {{ ref('model') }}            → model
   - {{ source('src', 'table') }}  → src__table
   - {{ config(...) }}             → (removed)
-  - {{ var('name') }}             → '__var_name__'
+  - {{ var('name') }}             → '__var_name__'  (string literal; may break
+                                     identifier contexts like FROM {{ var('t') }})
   - {{ var('name', default) }}    → '__var_name__'
   - {{ this }}                    → __this__
   - {{ this.identifier }}         → __this__
@@ -199,9 +200,10 @@ def preprocess_jinja(sql: str) -> JinjaPreprocessResult:
     # --- Pass 8: Stub this ---
     out = _RE_THIS.sub("__this__", out)
 
-    # --- Pass 9: Remove adapter.dispatch, return, log, exceptions ---
-    out = _RE_ADAPTER_DISPATCH.sub("", out)
-    out = _RE_UTILITY_CALLS.sub("", out)
+    # --- Pass 9: Replace adapter.dispatch, return, log, exceptions with placeholder ---
+    # Use __jinja_expr__ instead of empty string to avoid invalid SQL in expression positions
+    out = _RE_ADAPTER_DISPATCH.sub("__jinja_expr__", out)
+    out = _RE_UTILITY_CALLS.sub("__jinja_expr__", out)
 
     # --- Pass 10: Strip block tags (if/elif/else/endif, for/endfor) — keep content ---
     out = _RE_IF_OPEN.sub("", out)

packages/altimate-engine/tests/test_jinja_preprocessor.py

@@ -402,12 +402,14 @@ def test_adapter_dispatch(self):
         """
         result = preprocess_jinja(sql)
         assert "adapter" not in result.preprocessed_sql
+        assert "__jinja_expr__" in result.preprocessed_sql
         assert "SELECT * FROM orders" in result.preprocessed_sql
 
     def test_return(self):
         sql = "{{ return([]) }}"
         result = preprocess_jinja(sql)
         assert "return" not in result.preprocessed_sql
+        assert "__jinja_expr__" in result.preprocessed_sql
 
     def test_log(self):
         sql = """
@@ -416,6 +418,7 @@ def test_log(self):
         """
         result = preprocess_jinja(sql)
         assert "log(" not in result.preprocessed_sql
+        assert "__jinja_expr__" in result.preprocessed_sql
 
 
 # ---------------------------------------------------------------------------