diff --git a/macros/edr/metadata_collection/get_columns_from_information_schema.sql b/macros/edr/metadata_collection/get_columns_from_information_schema.sql index 723585beb..526d360f1 100644 --- a/macros/edr/metadata_collection/get_columns_from_information_schema.sql +++ b/macros/edr/metadata_collection/get_columns_from_information_schema.sql @@ -124,6 +124,18 @@ {{ elementary.get_empty_columns_from_information_schema_table() }} {% endmacro %} +{% macro fabricspark__get_columns_from_information_schema( + database_name, schema_name, table_name=none +) %} + {{ + return( + elementary.spark__get_columns_from_information_schema( + database_name, schema_name, table_name + ) + ) + }} +{% endmacro %} + {% macro get_empty_columns_from_information_schema_table() %} {{ elementary.empty_table( diff --git a/macros/edr/system/system_utils/buckets_cte.sql b/macros/edr/system/system_utils/buckets_cte.sql index a775b40f0..9902fdb87 100644 --- a/macros/edr/system/system_utils/buckets_cte.sql +++ b/macros/edr/system/system_utils/buckets_cte.sql @@ -84,6 +84,14 @@ {{ return(complete_buckets_cte) }} {% endmacro %} +{% macro fabricspark__complete_buckets_cte( + time_bucket, bucket_end_expr, min_bucket_start_expr, max_bucket_end_expr +) %} + {{ return(elementary.spark__complete_buckets_cte( + time_bucket, bucket_end_expr, min_bucket_start_expr, max_bucket_end_expr + )) }} +{% endmacro %} + {% macro snowflake__complete_buckets_cte( time_bucket, bucket_end_expr, diff --git a/macros/edr/tests/test_utils/clean_elementary_test_tables.sql b/macros/edr/tests/test_utils/clean_elementary_test_tables.sql index e5f600991..a1c8b1fd8 100644 --- a/macros/edr/tests/test_utils/clean_elementary_test_tables.sql +++ b/macros/edr/tests/test_utils/clean_elementary_test_tables.sql @@ -65,6 +65,16 @@ ) %} {% endmacro %} +{% macro fabricspark__get_clean_elementary_test_tables_queries(test_table_relations) %} + {{ + return( + elementary.spark__get_clean_elementary_test_tables_queries( + test_table_relations + ) + ) + }} +{% endmacro %} + {% macro clickhouse__get_clean_elementary_test_tables_queries(test_table_relations) %} {# Self-hosted clustered ClickHouse installations require tables to be dropped on all cluster nodes explicitly #} {% set queries = [] %} diff --git a/macros/utils/cross_db_utils/current_timestamp.sql b/macros/utils/cross_db_utils/current_timestamp.sql index af9b8cd6c..e438feae6 100644 --- a/macros/utils/cross_db_utils/current_timestamp.sql +++ b/macros/utils/cross_db_utils/current_timestamp.sql @@ -92,6 +92,14 @@ cast(sysutcdatetime() as datetime2(6)) {%- endmacro -%} +{% macro fabricspark__edr_current_timestamp() %} + {{ return(elementary.spark__edr_current_timestamp()) }} +{% endmacro %} + +{% macro fabricspark__edr_current_timestamp_in_utc() %} + {{ return(elementary.spark__edr_current_timestamp_in_utc()) }} +{% endmacro %} + {% macro dremio__edr_current_timestamp() -%} current_timestamp() {%- endmacro -%} {% macro dremio__edr_current_timestamp_in_utc() -%} diff --git a/macros/utils/cross_db_utils/datediff.sql b/macros/utils/cross_db_utils/datediff.sql index c66b35778..922e703c7 100644 --- a/macros/utils/cross_db_utils/datediff.sql +++ b/macros/utils/cross_db_utils/datediff.sql @@ -240,6 +240,10 @@ {% endmacro %} +{% macro fabricspark__edr_datediff(first_date, second_date, datepart) %} + {{ return(elementary.spark__edr_datediff(first_date, second_date, datepart)) }} +{% endmacro %} + {% macro athena__edr_datediff(first_date, second_date, date_part) %} {% set macro = dbt.datediff or dbt_utils.datediff %} {% if not macro %} diff --git a/macros/utils/cross_db_utils/generate_elementary_profile_args.sql b/macros/utils/cross_db_utils/generate_elementary_profile_args.sql index fdfb54156..b7c405991 100644 --- a/macros/utils/cross_db_utils/generate_elementary_profile_args.sql +++ b/macros/utils/cross_db_utils/generate_elementary_profile_args.sql @@ -311,6 +311,26 @@ ) %} {% endmacro %} +{% macro fabricspark__generate_elementary_profile_args( + method, elementary_database, elementary_schema +) %} + {% set parameters = [ + _parameter("type", "fabricspark"), + _parameter("method", "livy"), + _parameter("authentication", "CLI"), + _parameter( + "endpoint", + target.endpoint | default("https://api.fabric.microsoft.com/v1"), + ), + _parameter("workspaceid", ""), + _parameter("lakehouseid", ""), + _parameter("lakehouse", elementary_database), + _parameter("schema", elementary_schema), + _parameter("threads", target.threads), + ] %} + {% do return(parameters) %} +{% endmacro %} + {% macro default__generate_elementary_profile_args( method, elementary_database, elementary_schema ) %} diff --git a/macros/utils/cross_db_utils/incremental_strategy.sql b/macros/utils/cross_db_utils/incremental_strategy.sql index ed3391fa9..dfa6e292a 100644 --- a/macros/utils/cross_db_utils/incremental_strategy.sql +++ b/macros/utils/cross_db_utils/incremental_strategy.sql @@ -24,6 +24,10 @@ {% do return("merge") %} {% endmacro %} +{%- macro fabricspark__get_default_incremental_strategy() %} + {{ return(elementary.spark__get_default_incremental_strategy()) }} +{% endmacro %} + {% macro default__get_default_incremental_strategy() %} {% do return(none) %} {% endmacro %} diff --git a/macros/utils/cross_db_utils/safe_cast.sql b/macros/utils/cross_db_utils/safe_cast.sql index 0717e152d..501232804 100644 --- a/macros/utils/cross_db_utils/safe_cast.sql +++ b/macros/utils/cross_db_utils/safe_cast.sql @@ -13,3 +13,7 @@ {% macro spark__edr_safe_cast(field, type) %} try_cast({{ field }} as {{ type }}) {% endmacro %} + +{% macro fabricspark__edr_safe_cast(field, type) %} + {{ return(elementary.spark__edr_safe_cast(field, type)) }} +{% endmacro %} diff --git a/macros/utils/cross_db_utils/target_database.sql b/macros/utils/cross_db_utils/target_database.sql index e1653732b..45793ab8b 100644 --- a/macros/utils/cross_db_utils/target_database.sql +++ b/macros/utils/cross_db_utils/target_database.sql @@ -28,3 +28,7 @@ {% macro sqlserver__target_database() %} {% do return(target.database) %} {% endmacro %} {% macro vertica__target_database() %} {% do return(target.database) %} {% endmacro %} + +{% macro fabricspark__target_database() %} + {{ return(elementary.spark__target_database()) }} +{% endmacro %} diff --git a/macros/utils/cross_db_utils/to_char.sql b/macros/utils/cross_db_utils/to_char.sql index ae9e40347..8cde8b631 100644 --- a/macros/utils/cross_db_utils/to_char.sql +++ b/macros/utils/cross_db_utils/to_char.sql @@ -24,6 +24,10 @@ {%- endif %} {% endmacro %} +{% macro fabricspark__edr_to_char(column, format) %} + {{ return(elementary.spark__edr_to_char(column, format)) }} +{% endmacro %} + {% macro fabric__edr_to_char(column, format) %} convert(varchar, {{ column }} {%- if format %}, {{ format }}) diff --git a/macros/utils/data_types/data_type.sql b/macros/utils/data_types/data_type.sql index 0dc45d9b3..4b9e519f2 100644 --- a/macros/utils/data_types/data_type.sql +++ b/macros/utils/data_types/data_type.sql @@ -42,6 +42,10 @@ {% macro spark__edr_type_string() %} {% do return("string") %} {% endmacro %} +{% macro fabricspark__edr_type_string() %} + {{ return(elementary.spark__edr_type_string()) }} +{% endmacro %} + {% macro athena__edr_type_string() %} {% do return("varchar") %} {% endmacro %} {% macro trino__edr_type_string() %} {% do return("varchar") %} {% endmacro %} @@ -154,3 +158,11 @@ {% macro dremio__edr_type_timestamp() %} timestamp {% endmacro %} {% macro fabric__edr_type_timestamp() %} datetime2(6) {% endmacro %} + +{% macro fabricspark__edr_type_bool() %} + {{ return(elementary.default__edr_type_bool()) }} +{% endmacro %} + +{% macro fabricspark__edr_type_timestamp() %} + {{ return(elementary.default__edr_type_timestamp()) }} +{% endmacro %} diff --git a/macros/utils/data_types/data_type_list.sql b/macros/utils/data_types/data_type_list.sql index 0b471fe2c..1e14eaa20 100644 --- a/macros/utils/data_types/data_type_list.sql +++ b/macros/utils/data_types/data_type_list.sql @@ -146,6 +146,10 @@ {% endmacro %} +{% macro fabricspark__data_type_list(data_type) %} + {{ return(elementary.spark__data_type_list(data_type)) }} +{% endmacro %} + {% macro athena__data_type_list(data_type) %} {% set string_list = ["string", "varchar", "char"] | list %} diff --git a/macros/utils/data_types/get_normalized_data_type.sql b/macros/utils/data_types/get_normalized_data_type.sql index aca2baa03..ef8ca5a2d 100644 --- a/macros/utils/data_types/get_normalized_data_type.sql +++ b/macros/utils/data_types/get_normalized_data_type.sql @@ -69,6 +69,10 @@ {% endmacro %} +{% macro fabricspark__get_normalized_data_type(exact_data_type) %} + {{ return(elementary.spark__get_normalized_data_type(exact_data_type)) }} +{% endmacro %} + {% macro redshift__get_normalized_data_type(exact_data_type) %} {# understanding Redshift data type synonyms: https://docs.aws.amazon.com/redshift/latest/dg/c_Supported_data_types.html #} diff --git a/macros/utils/table_operations/create_or_replace.sql b/macros/utils/table_operations/create_or_replace.sql index 73a17d353..80c1f4cd6 100644 --- a/macros/utils/table_operations/create_or_replace.sql +++ b/macros/utils/table_operations/create_or_replace.sql @@ -33,6 +33,10 @@ ) %} {% endmacro %} +{% macro fabricspark__create_or_replace(temporary, relation, sql_query) %} + {{ return(elementary.spark__create_or_replace(temporary, relation, sql_query)) }} +{% endmacro %} + {% macro athena__create_or_replace(temporary, relation, sql_query) %} {% do elementary.edr_create_table_as( temporary, relation, sql_query, drop_first=true diff --git a/macros/utils/table_operations/create_table_as.sql b/macros/utils/table_operations/create_table_as.sql index 5e74e2c8a..acfd1d3ea 100644 --- a/macros/utils/table_operations/create_table_as.sql +++ b/macros/utils/table_operations/create_table_as.sql @@ -110,3 +110,13 @@ as {{ sql_query }} {% endif %} {% endmacro %} + +{% macro fabricspark__edr_get_create_table_as_sql(temporary, relation, sql_query) %} + {{ + return( + elementary.spark__edr_get_create_table_as_sql( + temporary, relation, sql_query + ) + ) + }} +{% endmacro %} diff --git a/macros/utils/table_operations/delete_and_insert.sql b/macros/utils/table_operations/delete_and_insert.sql index 542a0e979..7c9639f2a 100644 --- a/macros/utils/table_operations/delete_and_insert.sql +++ b/macros/utils/table_operations/delete_and_insert.sql @@ -143,6 +143,18 @@ {% do return(queries) %} {% endmacro %} +{% macro fabricspark__get_delete_and_insert_queries( + relation, insert_relation, delete_relation, delete_column_key +) %} + {{ + return( + elementary.spark__get_delete_and_insert_queries( + relation, insert_relation, delete_relation, delete_column_key + ) + ) + }} +{% endmacro %} + {% macro redshift__get_delete_and_insert_queries( relation, insert_relation, delete_relation, delete_column_key ) %} diff --git a/macros/utils/table_operations/get_relation_max_length.sql b/macros/utils/table_operations/get_relation_max_length.sql index 02e9577f1..d6e47500b 100644 --- a/macros/utils/table_operations/get_relation_max_length.sql +++ b/macros/utils/table_operations/get_relation_max_length.sql @@ -23,6 +23,16 @@ {{ return(127) }} {% endmacro %} +{% macro fabricspark__get_relation_max_name_length(temporary, relation, sql_query) %} + {{ + return( + elementary.spark__get_relation_max_name_length( + temporary, relation, sql_query + ) + ) + }} +{% endmacro %} + {% macro athena__get_relation_max_name_length(temporary, relation, sql_query) %} {{ return(255) }} {% endmacro %} diff --git a/macros/utils/table_operations/has_temp_table_support.sql b/macros/utils/table_operations/has_temp_table_support.sql index 9fe3fc715..988267675 100644 --- a/macros/utils/table_operations/has_temp_table_support.sql +++ b/macros/utils/table_operations/has_temp_table_support.sql @@ -6,6 +6,8 @@ {% macro spark__has_temp_table_support() %} {% do return(false) %} {% endmacro %} +{% macro fabricspark__has_temp_table_support() %} {% do return(false) %} {% endmacro %} + {% macro trino__has_temp_table_support() %} {% do return(false) %} {% endmacro %} {% macro athena__has_temp_table_support() %} {% do return(false) %} {% endmacro %} diff --git a/macros/utils/table_operations/insert_rows.sql b/macros/utils/table_operations/insert_rows.sql index 28ade72da..512c4b05b 100644 --- a/macros/utils/table_operations/insert_rows.sql +++ b/macros/utils/table_operations/insert_rows.sql @@ -309,6 +309,10 @@ {%- endif -%} {%- endmacro -%} +{%- macro fabricspark__escape_special_chars(string_value) -%} + {{- return(elementary.spark__escape_special_chars(string_value)) -}} +{%- endmacro -%} + {# Note: Python booleans pass Jinja's "is number" test, so we check "is boolean" first. edr_boolean_literal renders the correct SQL literal per adapter (TRUE/FALSE for standard SQL, cast(1/0 as bit) for T-SQL). #} diff --git a/macros/utils/table_operations/make_temp_relation.sql b/macros/utils/table_operations/make_temp_relation.sql index 42116f69f..c1d3f2553 100644 --- a/macros/utils/table_operations/make_temp_relation.sql +++ b/macros/utils/table_operations/make_temp_relation.sql @@ -64,6 +64,10 @@ {% do return(tmp_relation) %} {% endmacro %} +{% macro fabricspark__edr_make_temp_relation(base_relation, suffix) %} + {{ return(elementary.spark__edr_make_temp_relation(base_relation, suffix)) }} +{% endmacro %} + {% macro redshift__edr_make_temp_relation(base_relation, suffix) %} {% if elementary.is_dbt_fusion() %} {# Workaround for dbt-fusion temp table metadata bug - create regular relations diff --git a/macros/utils/table_operations/replace_table_data.sql b/macros/utils/table_operations/replace_table_data.sql index 50d3ab2f4..5ee153d50 100644 --- a/macros/utils/table_operations/replace_table_data.sql +++ b/macros/utils/table_operations/replace_table_data.sql @@ -37,6 +37,11 @@ ) %} {% endmacro %} +{# FabricSpark - delegate to Spark (non-atomic) #} +{% macro fabricspark__replace_table_data(relation, rows) %} + {{ return(elementary.spark__replace_table_data(relation, rows)) }} +{% endmacro %} + {# Dremio - truncate and insert (non-atomic) #} {% macro dremio__replace_table_data(relation, rows) %} {% do dbt.truncate_relation(relation) %}