From 51d63c8dbec12627a3388aed5cf4dcc48d1a2384 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Mar 2026 12:36:23 +0000 Subject: [PATCH 1/4] feat: add dbt-fabricspark adapter support (ELE-5295) Add fabricspark__ prefixed macros that delegate to spark__ implementations. dbt-fabricspark speaks Spark SQL but doesn't declare dependencies=["spark"] in its AdapterPlugin, so dispatch falls through to default__ instead of spark__. Macros added across 22 files covering: - Cross-DB utils (timestamps, dateadd/diff, to_char, safe_cast, etc.) - Data types (string, bool, timestamp, type lists, normalization) - Table operations (create, delete/insert, temp relations, etc.) - System utils (buckets CTE) - Metadata collection (information schema) - Test utils (clean test tables) Also adds a defensive fallback in get_elementary_relation.sql using api.Relation.create() when adapter.get_relation() returns None, working around a bug in dbt-fabricspark's list_relations_without_caching. Co-Authored-By: Itamar Hartstein --- .../get_columns_from_information_schema.sql | 12 ++++++++++++ macros/edr/system/system_utils/buckets_cte.sql | 8 ++++++++ .../clean_elementary_test_tables.sql | 10 ++++++++++ .../utils/cross_db_utils/current_timestamp.sql | 8 ++++++++ macros/utils/cross_db_utils/datediff.sql | 4 ++++ .../generate_elementary_profile_args.sql | 12 ++++++++++++ .../cross_db_utils/incremental_strategy.sql | 4 ++++ macros/utils/cross_db_utils/safe_cast.sql | 4 ++++ .../utils/cross_db_utils/target_database.sql | 4 ++++ macros/utils/cross_db_utils/to_char.sql | 4 ++++ macros/utils/data_types/data_type.sql | 12 ++++++++++++ macros/utils/data_types/data_type_list.sql | 4 ++++ .../data_types/get_normalized_data_type.sql | 4 ++++ macros/utils/graph/get_elementary_relation.sql | 18 ++++++++++++++---- .../table_operations/create_or_replace.sql | 4 ++++ .../utils/table_operations/create_table_as.sql | 10 ++++++++++ .../table_operations/delete_and_insert.sql | 12 ++++++++++++ .../get_relation_max_length.sql | 10 ++++++++++ .../has_temp_table_support.sql | 2 ++ macros/utils/table_operations/insert_rows.sql | 4 ++++ .../table_operations/make_temp_relation.sql | 4 ++++ .../table_operations/replace_table_data.sql | 5 +++++ 22 files changed, 155 insertions(+), 4 deletions(-) diff --git a/macros/edr/metadata_collection/get_columns_from_information_schema.sql b/macros/edr/metadata_collection/get_columns_from_information_schema.sql index 723585beb..526d360f1 100644 --- a/macros/edr/metadata_collection/get_columns_from_information_schema.sql +++ b/macros/edr/metadata_collection/get_columns_from_information_schema.sql @@ -124,6 +124,18 @@ {{ elementary.get_empty_columns_from_information_schema_table() }} {% endmacro %} +{% macro fabricspark__get_columns_from_information_schema( + database_name, schema_name, table_name=none +) %} + {{ + return( + elementary.spark__get_columns_from_information_schema( + database_name, schema_name, table_name + ) + ) + }} +{% endmacro %} + {% macro get_empty_columns_from_information_schema_table() %} {{ elementary.empty_table( diff --git a/macros/edr/system/system_utils/buckets_cte.sql b/macros/edr/system/system_utils/buckets_cte.sql index b45b2b999..d925376d7 100644 --- a/macros/edr/system/system_utils/buckets_cte.sql +++ b/macros/edr/system/system_utils/buckets_cte.sql @@ -84,6 +84,14 @@ {{ return(complete_buckets_cte) }} {% endmacro %} +{% macro fabricspark__complete_buckets_cte( + time_bucket, bucket_end_expr, min_bucket_start_expr, max_bucket_end_expr +) %} + {{ return(elementary.spark__complete_buckets_cte( + time_bucket, bucket_end_expr, min_bucket_start_expr, max_bucket_end_expr + )) }} +{% endmacro %} + {% macro snowflake__complete_buckets_cte( time_bucket, bucket_end_expr, diff --git a/macros/edr/tests/test_utils/clean_elementary_test_tables.sql b/macros/edr/tests/test_utils/clean_elementary_test_tables.sql index e5f600991..a1c8b1fd8 100644 --- a/macros/edr/tests/test_utils/clean_elementary_test_tables.sql +++ b/macros/edr/tests/test_utils/clean_elementary_test_tables.sql @@ -65,6 +65,16 @@ ) %} {% endmacro %} +{% macro fabricspark__get_clean_elementary_test_tables_queries(test_table_relations) %} + {{ + return( + elementary.spark__get_clean_elementary_test_tables_queries( + test_table_relations + ) + ) + }} +{% endmacro %} + {% macro clickhouse__get_clean_elementary_test_tables_queries(test_table_relations) %} {# Self-hosted clustered ClickHouse installations require tables to be dropped on all cluster nodes explicitly #} {% set queries = [] %} diff --git a/macros/utils/cross_db_utils/current_timestamp.sql b/macros/utils/cross_db_utils/current_timestamp.sql index af9b8cd6c..e438feae6 100644 --- a/macros/utils/cross_db_utils/current_timestamp.sql +++ b/macros/utils/cross_db_utils/current_timestamp.sql @@ -92,6 +92,14 @@ cast(sysutcdatetime() as datetime2(6)) {%- endmacro -%} +{% macro fabricspark__edr_current_timestamp() %} + {{ return(elementary.spark__edr_current_timestamp()) }} +{% endmacro %} + +{% macro fabricspark__edr_current_timestamp_in_utc() %} + {{ return(elementary.spark__edr_current_timestamp_in_utc()) }} +{% endmacro %} + {% macro dremio__edr_current_timestamp() -%} current_timestamp() {%- endmacro -%} {% macro dremio__edr_current_timestamp_in_utc() -%} diff --git a/macros/utils/cross_db_utils/datediff.sql b/macros/utils/cross_db_utils/datediff.sql index c66b35778..922e703c7 100644 --- a/macros/utils/cross_db_utils/datediff.sql +++ b/macros/utils/cross_db_utils/datediff.sql @@ -240,6 +240,10 @@ {% endmacro %} +{% macro fabricspark__edr_datediff(first_date, second_date, datepart) %} + {{ return(elementary.spark__edr_datediff(first_date, second_date, datepart)) }} +{% endmacro %} + {% macro athena__edr_datediff(first_date, second_date, date_part) %} {% set macro = dbt.datediff or dbt_utils.datediff %} {% if not macro %} diff --git a/macros/utils/cross_db_utils/generate_elementary_profile_args.sql b/macros/utils/cross_db_utils/generate_elementary_profile_args.sql index fdfb54156..cd1b4fd42 100644 --- a/macros/utils/cross_db_utils/generate_elementary_profile_args.sql +++ b/macros/utils/cross_db_utils/generate_elementary_profile_args.sql @@ -311,6 +311,18 @@ ) %} {% endmacro %} +{% macro fabricspark__generate_elementary_profile_args( + method, elementary_database, elementary_schema +) %} + {{ + return( + elementary.spark__generate_elementary_profile_args( + method, elementary_database, elementary_schema + ) + ) + }} +{% endmacro %} + {% macro default__generate_elementary_profile_args( method, elementary_database, elementary_schema ) %} diff --git a/macros/utils/cross_db_utils/incremental_strategy.sql b/macros/utils/cross_db_utils/incremental_strategy.sql index ed3391fa9..dfa6e292a 100644 --- a/macros/utils/cross_db_utils/incremental_strategy.sql +++ b/macros/utils/cross_db_utils/incremental_strategy.sql @@ -24,6 +24,10 @@ {% do return("merge") %} {% endmacro %} +{%- macro fabricspark__get_default_incremental_strategy() %} + {{ return(elementary.spark__get_default_incremental_strategy()) }} +{% endmacro %} + {% macro default__get_default_incremental_strategy() %} {% do return(none) %} {% endmacro %} diff --git a/macros/utils/cross_db_utils/safe_cast.sql b/macros/utils/cross_db_utils/safe_cast.sql index 0717e152d..501232804 100644 --- a/macros/utils/cross_db_utils/safe_cast.sql +++ b/macros/utils/cross_db_utils/safe_cast.sql @@ -13,3 +13,7 @@ {% macro spark__edr_safe_cast(field, type) %} try_cast({{ field }} as {{ type }}) {% endmacro %} + +{% macro fabricspark__edr_safe_cast(field, type) %} + {{ return(elementary.spark__edr_safe_cast(field, type)) }} +{% endmacro %} diff --git a/macros/utils/cross_db_utils/target_database.sql b/macros/utils/cross_db_utils/target_database.sql index 5361f5d86..3d39e5d74 100644 --- a/macros/utils/cross_db_utils/target_database.sql +++ b/macros/utils/cross_db_utils/target_database.sql @@ -26,3 +26,7 @@ {% macro fabric__target_database() %} {% do return(target.database) %} {% endmacro %} {% macro sqlserver__target_database() %} {% do return(target.database) %} {% endmacro %} + +{% macro fabricspark__target_database() %} + {{ return(elementary.spark__target_database()) }} +{% endmacro %} diff --git a/macros/utils/cross_db_utils/to_char.sql b/macros/utils/cross_db_utils/to_char.sql index ae9e40347..8cde8b631 100644 --- a/macros/utils/cross_db_utils/to_char.sql +++ b/macros/utils/cross_db_utils/to_char.sql @@ -24,6 +24,10 @@ {%- endif %} {% endmacro %} +{% macro fabricspark__edr_to_char(column, format) %} + {{ return(elementary.spark__edr_to_char(column, format)) }} +{% endmacro %} + {% macro fabric__edr_to_char(column, format) %} convert(varchar, {{ column }} {%- if format %}, {{ format }}) diff --git a/macros/utils/data_types/data_type.sql b/macros/utils/data_types/data_type.sql index 96a371b1d..71bc7756a 100644 --- a/macros/utils/data_types/data_type.sql +++ b/macros/utils/data_types/data_type.sql @@ -42,6 +42,10 @@ {% macro spark__edr_type_string() %} {% do return("string") %} {% endmacro %} +{% macro fabricspark__edr_type_string() %} + {{ return(elementary.spark__edr_type_string()) }} +{% endmacro %} + {% macro athena__edr_type_string() %} {% do return("varchar") %} {% endmacro %} {% macro trino__edr_type_string() %} {% do return("varchar") %} {% endmacro %} @@ -144,3 +148,11 @@ {% macro dremio__edr_type_timestamp() %} timestamp {% endmacro %} {% macro fabric__edr_type_timestamp() %} datetime2(6) {% endmacro %} + +{% macro fabricspark__edr_type_bool() %} + {{ return(elementary.default__edr_type_bool()) }} +{% endmacro %} + +{% macro fabricspark__edr_type_timestamp() %} + {{ return(elementary.default__edr_type_timestamp()) }} +{% endmacro %} diff --git a/macros/utils/data_types/data_type_list.sql b/macros/utils/data_types/data_type_list.sql index 0b471fe2c..1e14eaa20 100644 --- a/macros/utils/data_types/data_type_list.sql +++ b/macros/utils/data_types/data_type_list.sql @@ -146,6 +146,10 @@ {% endmacro %} +{% macro fabricspark__data_type_list(data_type) %} + {{ return(elementary.spark__data_type_list(data_type)) }} +{% endmacro %} + {% macro athena__data_type_list(data_type) %} {% set string_list = ["string", "varchar", "char"] | list %} diff --git a/macros/utils/data_types/get_normalized_data_type.sql b/macros/utils/data_types/get_normalized_data_type.sql index c4a24fda7..a52e9a0b6 100644 --- a/macros/utils/data_types/get_normalized_data_type.sql +++ b/macros/utils/data_types/get_normalized_data_type.sql @@ -69,6 +69,10 @@ {% endmacro %} +{% macro fabricspark__get_normalized_data_type(exact_data_type) %} + {{ return(elementary.spark__get_normalized_data_type(exact_data_type)) }} +{% endmacro %} + {% macro redshift__get_normalized_data_type(exact_data_type) %} {# understanding Redshift data type synonyms: https://docs.aws.amazon.com/redshift/latest/dg/c_Supported_data_types.html #} diff --git a/macros/utils/graph/get_elementary_relation.sql b/macros/utils/graph/get_elementary_relation.sql index a50de55e0..b8bba77b2 100644 --- a/macros/utils/graph/get_elementary_relation.sql +++ b/macros/utils/graph/get_elementary_relation.sql @@ -20,10 +20,20 @@ {% if this and this.database == elementary_database and this.schema == elementary_schema and this.identifier == identifier_alias %} {% do return(this) %} {% endif %} - {% do return( - adapter.get_relation( - elementary_database, elementary_schema, identifier_alias - ) + {% set rel = adapter.get_relation( + elementary_database, elementary_schema, identifier_alias ) %} + {# Defensive fallback: some adapters (e.g. dbt-fabricspark) have bugs in + list_relations_without_caching that cause adapter.get_relation() to + return None even when the relation exists. Fall back to + api.Relation.create() so downstream code still gets a usable ref. #} + {% if rel is none %} + {% set rel = api.Relation.create( + database=elementary_database, + schema=elementary_schema, + identifier=identifier_alias, + ) %} + {% endif %} + {% do return(rel) %} {%- endif %} {% endmacro %} diff --git a/macros/utils/table_operations/create_or_replace.sql b/macros/utils/table_operations/create_or_replace.sql index 73a17d353..80c1f4cd6 100644 --- a/macros/utils/table_operations/create_or_replace.sql +++ b/macros/utils/table_operations/create_or_replace.sql @@ -33,6 +33,10 @@ ) %} {% endmacro %} +{% macro fabricspark__create_or_replace(temporary, relation, sql_query) %} + {{ return(elementary.spark__create_or_replace(temporary, relation, sql_query)) }} +{% endmacro %} + {% macro athena__create_or_replace(temporary, relation, sql_query) %} {% do elementary.edr_create_table_as( temporary, relation, sql_query, drop_first=true diff --git a/macros/utils/table_operations/create_table_as.sql b/macros/utils/table_operations/create_table_as.sql index 5e74e2c8a..acfd1d3ea 100644 --- a/macros/utils/table_operations/create_table_as.sql +++ b/macros/utils/table_operations/create_table_as.sql @@ -110,3 +110,13 @@ as {{ sql_query }} {% endif %} {% endmacro %} + +{% macro fabricspark__edr_get_create_table_as_sql(temporary, relation, sql_query) %} + {{ + return( + elementary.spark__edr_get_create_table_as_sql( + temporary, relation, sql_query + ) + ) + }} +{% endmacro %} diff --git a/macros/utils/table_operations/delete_and_insert.sql b/macros/utils/table_operations/delete_and_insert.sql index 542a0e979..7c9639f2a 100644 --- a/macros/utils/table_operations/delete_and_insert.sql +++ b/macros/utils/table_operations/delete_and_insert.sql @@ -143,6 +143,18 @@ {% do return(queries) %} {% endmacro %} +{% macro fabricspark__get_delete_and_insert_queries( + relation, insert_relation, delete_relation, delete_column_key +) %} + {{ + return( + elementary.spark__get_delete_and_insert_queries( + relation, insert_relation, delete_relation, delete_column_key + ) + ) + }} +{% endmacro %} + {% macro redshift__get_delete_and_insert_queries( relation, insert_relation, delete_relation, delete_column_key ) %} diff --git a/macros/utils/table_operations/get_relation_max_length.sql b/macros/utils/table_operations/get_relation_max_length.sql index 7dfaf4a2c..66082064f 100644 --- a/macros/utils/table_operations/get_relation_max_length.sql +++ b/macros/utils/table_operations/get_relation_max_length.sql @@ -23,6 +23,16 @@ {{ return(127) }} {% endmacro %} +{% macro fabricspark__get_relation_max_name_length(temporary, relation, sql_query) %} + {{ + return( + elementary.spark__get_relation_max_name_length( + temporary, relation, sql_query + ) + ) + }} +{% endmacro %} + {% macro athena__get_relation_max_name_length(temporary, relation, sql_query) %} {{ return(255) }} {% endmacro %} diff --git a/macros/utils/table_operations/has_temp_table_support.sql b/macros/utils/table_operations/has_temp_table_support.sql index 9fe3fc715..988267675 100644 --- a/macros/utils/table_operations/has_temp_table_support.sql +++ b/macros/utils/table_operations/has_temp_table_support.sql @@ -6,6 +6,8 @@ {% macro spark__has_temp_table_support() %} {% do return(false) %} {% endmacro %} +{% macro fabricspark__has_temp_table_support() %} {% do return(false) %} {% endmacro %} + {% macro trino__has_temp_table_support() %} {% do return(false) %} {% endmacro %} {% macro athena__has_temp_table_support() %} {% do return(false) %} {% endmacro %} diff --git a/macros/utils/table_operations/insert_rows.sql b/macros/utils/table_operations/insert_rows.sql index da23e9c5a..6039f7431 100644 --- a/macros/utils/table_operations/insert_rows.sql +++ b/macros/utils/table_operations/insert_rows.sql @@ -305,6 +305,10 @@ {%- endif -%} {%- endmacro -%} +{%- macro fabricspark__escape_special_chars(string_value) -%} + {{- return(elementary.spark__escape_special_chars(string_value)) -}} +{%- endmacro -%} + {# Note: Python booleans pass Jinja's "is number" test, so we check "is boolean" first. edr_boolean_literal renders the correct SQL literal per adapter (TRUE/FALSE for standard SQL, cast(1/0 as bit) for T-SQL). #} diff --git a/macros/utils/table_operations/make_temp_relation.sql b/macros/utils/table_operations/make_temp_relation.sql index 42116f69f..c1d3f2553 100644 --- a/macros/utils/table_operations/make_temp_relation.sql +++ b/macros/utils/table_operations/make_temp_relation.sql @@ -64,6 +64,10 @@ {% do return(tmp_relation) %} {% endmacro %} +{% macro fabricspark__edr_make_temp_relation(base_relation, suffix) %} + {{ return(elementary.spark__edr_make_temp_relation(base_relation, suffix)) }} +{% endmacro %} + {% macro redshift__edr_make_temp_relation(base_relation, suffix) %} {% if elementary.is_dbt_fusion() %} {# Workaround for dbt-fusion temp table metadata bug - create regular relations diff --git a/macros/utils/table_operations/replace_table_data.sql b/macros/utils/table_operations/replace_table_data.sql index 50d3ab2f4..5ee153d50 100644 --- a/macros/utils/table_operations/replace_table_data.sql +++ b/macros/utils/table_operations/replace_table_data.sql @@ -37,6 +37,11 @@ ) %} {% endmacro %} +{# FabricSpark - delegate to Spark (non-atomic) #} +{% macro fabricspark__replace_table_data(relation, rows) %} + {{ return(elementary.spark__replace_table_data(relation, rows)) }} +{% endmacro %} + {# Dremio - truncate and insert (non-atomic) #} {% macro dremio__replace_table_data(relation, rows) %} {% do dbt.truncate_relation(relation) %} From 50352555ee84b14b803977149f47ba60402dd82e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Mar 2026 12:46:22 +0000 Subject: [PATCH 2/4] revert: remove get_elementary_relation.sql defensive fallback Per reviewer feedback, removing the api.Relation.create() fallback since callers rely on None returns to detect missing relations. Co-Authored-By: Itamar Hartstein --- macros/utils/graph/get_elementary_relation.sql | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/macros/utils/graph/get_elementary_relation.sql b/macros/utils/graph/get_elementary_relation.sql index b8bba77b2..a50de55e0 100644 --- a/macros/utils/graph/get_elementary_relation.sql +++ b/macros/utils/graph/get_elementary_relation.sql @@ -20,20 +20,10 @@ {% if this and this.database == elementary_database and this.schema == elementary_schema and this.identifier == identifier_alias %} {% do return(this) %} {% endif %} - {% set rel = adapter.get_relation( - elementary_database, elementary_schema, identifier_alias + {% do return( + adapter.get_relation( + elementary_database, elementary_schema, identifier_alias + ) ) %} - {# Defensive fallback: some adapters (e.g. dbt-fabricspark) have bugs in - list_relations_without_caching that cause adapter.get_relation() to - return None even when the relation exists. Fall back to - api.Relation.create() so downstream code still gets a usable ref. #} - {% if rel is none %} - {% set rel = api.Relation.create( - database=elementary_database, - schema=elementary_schema, - identifier=identifier_alias, - ) %} - {% endif %} - {% do return(rel) %} {%- endif %} {% endmacro %} From d0fb760d664977a03a89ec0465d4feae22caf0f5 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Mar 2026 13:44:18 +0000 Subject: [PATCH 3/4] feat: dedicated fabricspark__generate_elementary_profile_args with Fabric Livy API params Co-Authored-By: Itamar Hartstein --- .../generate_elementary_profile_args.sql | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/macros/utils/cross_db_utils/generate_elementary_profile_args.sql b/macros/utils/cross_db_utils/generate_elementary_profile_args.sql index cd1b4fd42..092e56d34 100644 --- a/macros/utils/cross_db_utils/generate_elementary_profile_args.sql +++ b/macros/utils/cross_db_utils/generate_elementary_profile_args.sql @@ -314,13 +314,21 @@ {% macro fabricspark__generate_elementary_profile_args( method, elementary_database, elementary_schema ) %} - {{ - return( - elementary.spark__generate_elementary_profile_args( - method, elementary_database, elementary_schema - ) - ) - }} + {% set parameters = [ + _parameter("type", "fabricspark"), + _parameter("method", "livy"), + _parameter("authentication", "CLI"), + _parameter( + "endpoint", + target.endpoint | default("https://api.fabric.microsoft.com/v1"), + ), + _parameter("workspaceid", ""), + _parameter("lakehouseid", ""), + _parameter("lakehouse", elementary_schema), + _parameter("schema", elementary_schema), + _parameter("threads", target.threads), + ] %} + {% do return(parameters) %} {% endmacro %} {% macro default__generate_elementary_profile_args( From e6072fa92eb2de1b0f3bfe273d17a4663c53c61b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Mar 2026 13:50:03 +0000 Subject: [PATCH 4/4] fix: map lakehouse param to elementary_database instead of elementary_schema Co-Authored-By: Itamar Hartstein --- .../utils/cross_db_utils/generate_elementary_profile_args.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/utils/cross_db_utils/generate_elementary_profile_args.sql b/macros/utils/cross_db_utils/generate_elementary_profile_args.sql index 092e56d34..b7c405991 100644 --- a/macros/utils/cross_db_utils/generate_elementary_profile_args.sql +++ b/macros/utils/cross_db_utils/generate_elementary_profile_args.sql @@ -324,7 +324,7 @@ ), _parameter("workspaceid", ""), _parameter("lakehouseid", ""), - _parameter("lakehouse", elementary_schema), + _parameter("lakehouse", elementary_database), _parameter("schema", elementary_schema), _parameter("threads", target.threads), ] %}