feat: add dbt-fabricspark adapter support (ELE-5295) (#964)

devin-ai-integration[bot] · haritamar · web-flow · commit 5fafd8d92b3e · 2026-03-12T15:57:49.000+02:00
* feat: add dbt-fabricspark adapter support (ELE-5295)

Add fabricspark__ prefixed macros that delegate to spark__ implementations.
dbt-fabricspark speaks Spark SQL but doesn't declare dependencies=["spark"]
in its AdapterPlugin, so dispatch falls through to default__ instead of spark__.

Macros added across 22 files covering:
- Cross-DB utils (timestamps, dateadd/diff, to_char, safe_cast, etc.)
- Data types (string, bool, timestamp, type lists, normalization)
- Table operations (create, delete/insert, temp relations, etc.)
- System utils (buckets CTE)
- Metadata collection (information schema)
- Test utils (clean test tables)

Also adds a defensive fallback in get_elementary_relation.sql using
api.Relation.create() when adapter.get_relation() returns None, working
around a bug in dbt-fabricspark's list_relations_without_caching.

Co-Authored-By: Itamar Hartstein &lt;haritamar@gmail.com&gt;

* revert: remove get_elementary_relation.sql defensive fallback

Per reviewer feedback, removing the api.Relation.create() fallback
since callers rely on None returns to detect missing relations.

Co-Authored-By: Itamar Hartstein &lt;haritamar@gmail.com&gt;

* feat: dedicated fabricspark__generate_elementary_profile_args with Fabric Livy API params

Co-Authored-By: Itamar Hartstein &lt;haritamar@gmail.com&gt;

* fix: map lakehouse param to elementary_database instead of elementary_schema

Co-Authored-By: Itamar Hartstein &lt;haritamar@gmail.com&gt;

---------

Co-authored-by: Devin AI &lt;158243242+devin-ai-integration[bot]@users.noreply.github.com&gt;
Co-authored-by: Itamar Hartstein &lt;haritamar@gmail.com&gt;
diff --git a/macros/edr/metadata_collection/get_columns_from_information_schema.sql b/macros/edr/metadata_collection/get_columns_from_information_schema.sql
@@ -124,6 +124,18 @@
     {{ elementary.get_empty_columns_from_information_schema_table() }}
 {% endmacro %}
 
+{% macro fabricspark__get_columns_from_information_schema(
+    database_name, schema_name, table_name=none
+) %}
+    {{
+        return(
+            elementary.spark__get_columns_from_information_schema(
+                database_name, schema_name, table_name
+            )
+        )
+    }}
+{% endmacro %}
+
 {% macro get_empty_columns_from_information_schema_table() %}
     {{
         elementary.empty_table(
diff --git a/macros/edr/system/system_utils/buckets_cte.sql b/macros/edr/system/system_utils/buckets_cte.sql
@@ -84,6 +84,14 @@
     {{ return(complete_buckets_cte) }}
 {% endmacro %}
 
+{% macro fabricspark__complete_buckets_cte(
+    time_bucket, bucket_end_expr, min_bucket_start_expr, max_bucket_end_expr
+) %}
+    {{ return(elementary.spark__complete_buckets_cte(
+        time_bucket, bucket_end_expr, min_bucket_start_expr, max_bucket_end_expr
+    )) }}
+{% endmacro %}
+
 {% macro snowflake__complete_buckets_cte(
     time_bucket,
     bucket_end_expr,
diff --git a/macros/edr/tests/test_utils/clean_elementary_test_tables.sql b/macros/edr/tests/test_utils/clean_elementary_test_tables.sql
@@ -65,6 +65,16 @@
     ) %}
 {% endmacro %}
 
+{% macro fabricspark__get_clean_elementary_test_tables_queries(test_table_relations) %}
+    {{
+        return(
+            elementary.spark__get_clean_elementary_test_tables_queries(
+                test_table_relations
+            )
+        )
+    }}
+{% endmacro %}
+
 {% macro clickhouse__get_clean_elementary_test_tables_queries(test_table_relations) %}
     {# Self-hosted clustered ClickHouse installations require tables to be dropped on all cluster nodes explicitly #}
     {% set queries = [] %}
diff --git a/macros/utils/cross_db_utils/current_timestamp.sql b/macros/utils/cross_db_utils/current_timestamp.sql
@@ -92,6 +92,14 @@
     cast(sysutcdatetime() as datetime2(6))
 {%- endmacro -%}
 
+{% macro fabricspark__edr_current_timestamp() %}
+    {{ return(elementary.spark__edr_current_timestamp()) }}
+{% endmacro %}
+
+{% macro fabricspark__edr_current_timestamp_in_utc() %}
+    {{ return(elementary.spark__edr_current_timestamp_in_utc()) }}
+{% endmacro %}
+
 {% macro dremio__edr_current_timestamp() -%} current_timestamp() {%- endmacro -%}
 
 {% macro dremio__edr_current_timestamp_in_utc() -%}
diff --git a/macros/utils/cross_db_utils/datediff.sql b/macros/utils/cross_db_utils/datediff.sql
@@ -240,6 +240,10 @@
 
 {% endmacro %}
 
+{% macro fabricspark__edr_datediff(first_date, second_date, datepart) %}
+    {{ return(elementary.spark__edr_datediff(first_date, second_date, datepart)) }}
+{% endmacro %}
+
 {% macro athena__edr_datediff(first_date, second_date, date_part) %}
     {% set macro = dbt.datediff or dbt_utils.datediff %}
     {% if not macro %}
diff --git a/macros/utils/cross_db_utils/generate_elementary_profile_args.sql b/macros/utils/cross_db_utils/generate_elementary_profile_args.sql
@@ -311,6 +311,26 @@
     ) %}
 {% endmacro %}
 
+{% macro fabricspark__generate_elementary_profile_args(
+    method, elementary_database, elementary_schema
+) %}
+    {% set parameters = [
+        _parameter("type", "fabricspark"),
+        _parameter("method", "livy"),
+        _parameter("authentication", "CLI"),
+        _parameter(
+            "endpoint",
+            target.endpoint | default("https://api.fabric.microsoft.com/v1"),
+        ),
+        _parameter("workspaceid", "<WORKSPACE_GUID>"),
+        _parameter("lakehouseid", "<LAKEHOUSE_GUID>"),
+        _parameter("lakehouse", elementary_database),
+        _parameter("schema", elementary_schema),
+        _parameter("threads", target.threads),
+    ] %}
+    {% do return(parameters) %}
+{% endmacro %}
+
 {% macro default__generate_elementary_profile_args(
     method, elementary_database, elementary_schema
 ) %}
diff --git a/macros/utils/cross_db_utils/incremental_strategy.sql b/macros/utils/cross_db_utils/incremental_strategy.sql
@@ -24,6 +24,10 @@
     {% do return("merge") %}
 {% endmacro %}
 
+{%- macro fabricspark__get_default_incremental_strategy() %}
+    {{ return(elementary.spark__get_default_incremental_strategy()) }}
+{% endmacro %}
+
 {% macro default__get_default_incremental_strategy() %}
     {% do return(none) %}
 {% endmacro %}
diff --git a/macros/utils/cross_db_utils/safe_cast.sql b/macros/utils/cross_db_utils/safe_cast.sql
@@ -13,3 +13,7 @@
 {% macro spark__edr_safe_cast(field, type) %}
     try_cast({{ field }} as {{ type }})
 {% endmacro %}
+
+{% macro fabricspark__edr_safe_cast(field, type) %}
+    {{ return(elementary.spark__edr_safe_cast(field, type)) }}
+{% endmacro %}
diff --git a/macros/utils/cross_db_utils/target_database.sql b/macros/utils/cross_db_utils/target_database.sql
@@ -28,3 +28,7 @@
 {% macro sqlserver__target_database() %} {% do return(target.database) %} {% endmacro %}
 
 {% macro vertica__target_database() %} {% do return(target.database) %} {% endmacro %}
+
+{% macro fabricspark__target_database() %}
+    {{ return(elementary.spark__target_database()) }}
+{% endmacro %}
diff --git a/macros/utils/cross_db_utils/to_char.sql b/macros/utils/cross_db_utils/to_char.sql
@@ -24,6 +24,10 @@
     {%- endif %}
 {% endmacro %}
 
+{% macro fabricspark__edr_to_char(column, format) %}
+    {{ return(elementary.spark__edr_to_char(column, format)) }}
+{% endmacro %}
+
 {% macro fabric__edr_to_char(column, format) %}
     convert(varchar, {{ column }}
     {%- if format %}, {{ format }})
diff --git a/macros/utils/data_types/data_type.sql b/macros/utils/data_types/data_type.sql
@@ -42,6 +42,10 @@
 
 {% macro spark__edr_type_string() %} {% do return("string") %} {% endmacro %}
 
+{% macro fabricspark__edr_type_string() %}
+    {{ return(elementary.spark__edr_type_string()) }}
+{% endmacro %}
+
 {% macro athena__edr_type_string() %} {% do return("varchar") %} {% endmacro %}
 
 {% macro trino__edr_type_string() %} {% do return("varchar") %} {% endmacro %}
@@ -154,3 +158,11 @@
 {% macro dremio__edr_type_timestamp() %} timestamp {% endmacro %}
 
 {% macro fabric__edr_type_timestamp() %} datetime2(6) {% endmacro %}
+
+{% macro fabricspark__edr_type_bool() %}
+    {{ return(elementary.default__edr_type_bool()) }}
+{% endmacro %}
+
+{% macro fabricspark__edr_type_timestamp() %}
+    {{ return(elementary.default__edr_type_timestamp()) }}
+{% endmacro %}
diff --git a/macros/utils/data_types/data_type_list.sql b/macros/utils/data_types/data_type_list.sql
@@ -146,6 +146,10 @@
 {% endmacro %}
 
 
+{% macro fabricspark__data_type_list(data_type) %}
+    {{ return(elementary.spark__data_type_list(data_type)) }}
+{% endmacro %}
+
 {% macro athena__data_type_list(data_type) %}
 
     {% set string_list = ["string", "varchar", "char"] | list %}
diff --git a/macros/utils/data_types/get_normalized_data_type.sql b/macros/utils/data_types/get_normalized_data_type.sql
@@ -69,6 +69,10 @@
 {% endmacro %}
 
 
+{% macro fabricspark__get_normalized_data_type(exact_data_type) %}
+    {{ return(elementary.spark__get_normalized_data_type(exact_data_type)) }}
+{% endmacro %}
+
 {% macro redshift__get_normalized_data_type(exact_data_type) %}
     {# understanding Redshift data type synonyms:
  https://docs.aws.amazon.com/redshift/latest/dg/c_Supported_data_types.html #}
diff --git a/macros/utils/table_operations/create_or_replace.sql b/macros/utils/table_operations/create_or_replace.sql
@@ -33,6 +33,10 @@
     ) %}
 {% endmacro %}
 
+{% macro fabricspark__create_or_replace(temporary, relation, sql_query) %}
+    {{ return(elementary.spark__create_or_replace(temporary, relation, sql_query)) }}
+{% endmacro %}
+
 {% macro athena__create_or_replace(temporary, relation, sql_query) %}
     {% do elementary.edr_create_table_as(
         temporary, relation, sql_query, drop_first=true
diff --git a/macros/utils/table_operations/create_table_as.sql b/macros/utils/table_operations/create_table_as.sql
@@ -110,3 +110,13 @@
     as {{ sql_query }}
     {% endif %}
 {% endmacro %}
+
+{% macro fabricspark__edr_get_create_table_as_sql(temporary, relation, sql_query) %}
+    {{
+        return(
+            elementary.spark__edr_get_create_table_as_sql(
+                temporary, relation, sql_query
+            )
+        )
+    }}
+{% endmacro %}
diff --git a/macros/utils/table_operations/delete_and_insert.sql b/macros/utils/table_operations/delete_and_insert.sql
@@ -143,6 +143,18 @@
     {% do return(queries) %}
 {% endmacro %}
 
+{% macro fabricspark__get_delete_and_insert_queries(
+    relation, insert_relation, delete_relation, delete_column_key
+) %}
+    {{
+        return(
+            elementary.spark__get_delete_and_insert_queries(
+                relation, insert_relation, delete_relation, delete_column_key
+            )
+        )
+    }}
+{% endmacro %}
+
 {% macro redshift__get_delete_and_insert_queries(
     relation, insert_relation, delete_relation, delete_column_key
 ) %}
diff --git a/macros/utils/table_operations/get_relation_max_length.sql b/macros/utils/table_operations/get_relation_max_length.sql
@@ -23,6 +23,16 @@
     {{ return(127) }}
 {% endmacro %}
 
+{% macro fabricspark__get_relation_max_name_length(temporary, relation, sql_query) %}
+    {{
+        return(
+            elementary.spark__get_relation_max_name_length(
+                temporary, relation, sql_query
+            )
+        )
+    }}
+{% endmacro %}
+
 {% macro athena__get_relation_max_name_length(temporary, relation, sql_query) %}
     {{ return(255) }}
 {% endmacro %}
diff --git a/macros/utils/table_operations/has_temp_table_support.sql b/macros/utils/table_operations/has_temp_table_support.sql
@@ -6,6 +6,8 @@
 
 {% macro spark__has_temp_table_support() %} {% do return(false) %} {% endmacro %}
 
+{% macro fabricspark__has_temp_table_support() %} {% do return(false) %} {% endmacro %}
+
 {% macro trino__has_temp_table_support() %} {% do return(false) %} {% endmacro %}
 
 {% macro athena__has_temp_table_support() %} {% do return(false) %} {% endmacro %}
diff --git a/macros/utils/table_operations/insert_rows.sql b/macros/utils/table_operations/insert_rows.sql
@@ -309,6 +309,10 @@
     {%- endif -%}
 {%- endmacro -%}
 
+{%- macro fabricspark__escape_special_chars(string_value) -%}
+    {{- return(elementary.spark__escape_special_chars(string_value)) -}}
+{%- endmacro -%}
+
 {# Note: Python booleans pass Jinja's "is number" test, so we check
    "is boolean" first. edr_boolean_literal renders the correct SQL literal
    per adapter (TRUE/FALSE for standard SQL, cast(1/0 as bit) for T-SQL). #}
diff --git a/macros/utils/table_operations/make_temp_relation.sql b/macros/utils/table_operations/make_temp_relation.sql
@@ -64,6 +64,10 @@
     {% do return(tmp_relation) %}
 {% endmacro %}
 
+{% macro fabricspark__edr_make_temp_relation(base_relation, suffix) %}
+    {{ return(elementary.spark__edr_make_temp_relation(base_relation, suffix)) }}
+{% endmacro %}
+
 {% macro redshift__edr_make_temp_relation(base_relation, suffix) %}
     {% if elementary.is_dbt_fusion() %}
         {# Workaround for dbt-fusion temp table metadata bug - create regular relations
diff --git a/macros/utils/table_operations/replace_table_data.sql b/macros/utils/table_operations/replace_table_data.sql
@@ -37,6 +37,11 @@
     ) %}
 {% endmacro %}
 
+{# FabricSpark - delegate to Spark (non-atomic) #}
+{% macro fabricspark__replace_table_data(relation, rows) %}
+    {{ return(elementary.spark__replace_table_data(relation, rows)) }}
+{% endmacro %}
+
 {# Dremio - truncate and insert (non-atomic) #}
 {% macro dremio__replace_table_data(relation, rows) %}
     {% do dbt.truncate_relation(relation) %}
