diff --git a/dbt_project.yml b/dbt_project.yml index 84b5b614..3dfc864a 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -33,6 +33,7 @@ vars: #Stage Configuration datavault4dbt.copy_rsrc_ldts_input_columns: false + datavault4dbt.max_days_for_late_arriving_data: #General Configuration datavault4dbt.include_business_objects_before_appearance: false diff --git a/macros/staging/snowflake/stage.sql b/macros/staging/snowflake/stage.sql index 500905df..1ad66540 100644 --- a/macros/staging/snowflake/stage.sql +++ b/macros/staging/snowflake/stage.sql @@ -54,6 +54,8 @@ {# Setting the column name for load date timestamp and record source to the alias coming from the attributes #} {%- set ldts_alias = var('datavault4dbt.ldts_alias', 'ldts') -%} {%- set rsrc_alias = var('datavault4dbt.rsrc_alias', 'rsrc') -%} +{%- set max_days_for_late_arriving_data = var('datavault4dbt.max_days_for_late_arriving_data', 'rsrc') -%} + {%- set copy_input_columns = var('datavault4dbt.copy_rsrc_ldts_input_columns', false) -%} {%- set load_datetime_col_name = ldts_alias -%} {%- set record_source_col_name = rsrc_alias -%} @@ -201,9 +203,14 @@ source_data AS ( {{- "\n\n " ~ datavault4dbt.print_list(datavault4dbt.escape_column_names(all_source_columns)) if all_source_columns else " *" }} FROM {{ source_relation }} + WHERE 1 = 1 + {% if max_days_for_late_arriving_data|int != 0 %} + + AND {{ ldts }} >= DATEADD(day, {{ max_days_for_late_arriving_data }}*-1, CURRENT_DATE) + {%- endif -%} {% if is_incremental() %} - WHERE {{ ldts }} > (SELECT max({{ load_datetime_col_name}}) + AND {{ ldts }} > (SELECT max({{ load_datetime_col_name}}) FROM {{ this }} WHERE {{ load_datetime_col_name}} != {{ datavault4dbt.string_to_timestamp(timestamp_format , end_of_all_times) }} ) {%- endif -%}