From 5a0104adb4cad81cedb08fc4b60779944e8fc758 Mon Sep 17 00:00:00 2001 From: JVBE Date: Thu, 15 May 2025 21:44:27 +0200 Subject: [PATCH 1/3] NOTICKET (feat): Add performance optimization for snowflake external tables by limiting the partitions based on ldts --- dbt_project.yml | 1 + macros/staging/snowflake/stage.sql | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 7871ce05..fcd9b6ab 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -33,6 +33,7 @@ vars: #Stage Configuration datavault4dbt.copy_rsrc_ldts_input_columns: false + datavault4dbt.max_days_for_late_arriving_data: 7 #General Configuration datavault4dbt.include_business_objects_before_appearance: false diff --git a/macros/staging/snowflake/stage.sql b/macros/staging/snowflake/stage.sql index 8276699f..70a13f14 100644 --- a/macros/staging/snowflake/stage.sql +++ b/macros/staging/snowflake/stage.sql @@ -1,4 +1,4 @@ -{# This is the default version of the stage macro, designed for Snowflake. #} +{# This is the default version of the stage macro, designed for Snowflake. Additionally a horizont of x days can be set to limit the amount of data and partitions that get loaded (Snowflake partition pruning does not work with subqueries) #} {%- macro snowflake__stage(include_source_columns, ldts, @@ -54,6 +54,8 @@ {# Setting the column name for load date timestamp and record source to the alias coming from the attributes #} {%- set ldts_alias = var('datavault4dbt.ldts_alias', 'ldts') -%} {%- set rsrc_alias = var('datavault4dbt.rsrc_alias', 'rsrc') -%} +{%- set max_days_for_late_arriving_data = var('datavault4dbt.max_days_for_late_arriving_data', 'rsrc') -%} + {%- set copy_input_columns = var('datavault4dbt.copy_rsrc_ldts_input_columns', false) -%} {%- set load_datetime_col_name = ldts_alias -%} {%- set record_source_col_name = rsrc_alias -%} @@ -201,9 +203,14 @@ source_data AS ( {{- "\n\n " ~ datavault4dbt.print_list(datavault4dbt.escape_column_names(all_source_columns)) if all_source_columns else " *" }} FROM {{ source_relation }} + WHERE 1 = 1 + {% if max_days_for_late_arriving_data|int != 0 %} + + AND {{ ldts }} >= DATEADD(day, {{ max_days_for_late_arriving_data }}*-1, CURRENT_DATE) + {%- endif -%} {% if is_incremental() %} - WHERE {{ ldts }} > (SELECT max({{ load_datetime_col_name}}) + AND {{ ldts }} > (SELECT max({{ load_datetime_col_name}}) FROM {{ this }} WHERE {{ load_datetime_col_name}} != {{ datavault4dbt.string_to_timestamp(timestamp_format , end_of_all_times) }} ) {%- endif -%} From e8c3009e10f838b48f3b264c47e634cb1adec316 Mon Sep 17 00:00:00 2001 From: JVBE Date: Thu, 15 May 2025 21:59:56 +0200 Subject: [PATCH 2/3] NOTICKET (fix): Revert comment change --- macros/staging/snowflake/stage.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/staging/snowflake/stage.sql b/macros/staging/snowflake/stage.sql index 70a13f14..bebcf97c 100644 --- a/macros/staging/snowflake/stage.sql +++ b/macros/staging/snowflake/stage.sql @@ -1,4 +1,4 @@ -{# This is the default version of the stage macro, designed for Snowflake. Additionally a horizont of x days can be set to limit the amount of data and partitions that get loaded (Snowflake partition pruning does not work with subqueries) #} +{# This is the default version of the stage macro, designed for Snowflake. #} {%- macro snowflake__stage(include_source_columns, ldts, From 7d2565b2ef0759f7ed87198d8ee131e58cdc59f6 Mon Sep 17 00:00:00 2001 From: JVBE Date: Thu, 15 May 2025 22:01:20 +0200 Subject: [PATCH 3/3] NOTICKET (fix): Unset variable for max_days_data_late to not break anything in default --- dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index fcd9b6ab..a5d24ae2 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -33,7 +33,7 @@ vars: #Stage Configuration datavault4dbt.copy_rsrc_ldts_input_columns: false - datavault4dbt.max_days_for_late_arriving_data: 7 + datavault4dbt.max_days_for_late_arriving_data: #General Configuration datavault4dbt.include_business_objects_before_appearance: false