diff --git a/macros/hooks/clean_up_pit.md b/macros/hooks/clean_up_pit.md new file mode 100644 index 00000000..668990ae --- /dev/null +++ b/macros/hooks/clean_up_pit.md @@ -0,0 +1,29 @@ +{% docs clean_up_pit %} + +## Clean Up PIT (Post-Hook) + +This macro should be used as a post-hook for each PIT table whenever a logarithmic snapshot logic is used. +The macro deletes all records in a PIT table that are no longer active according to the snapshot view. +Deletion is safe here because no actual data is deleted — only pointers to satellite entries. + +### Usage as a post-hook + +```jinja +{{ config( + post_hook="{{ datavault4dbt.clean_up_pit('control_snap_v1') }}" +) }} +``` + +### With custom column names + +```jinja +{{ config( + post_hook="{{ datavault4dbt.clean_up_pit( + snapshot_relation='control_snap_v1', + snapshot_trigger_column='is_active', + sdts='sdts' + ) }}" +) }} +``` + +{% enddocs %} diff --git a/macros/hooks/clean_up_pit.yml b/macros/hooks/clean_up_pit.yml new file mode 100644 index 00000000..713a0ef3 --- /dev/null +++ b/macros/hooks/clean_up_pit.yml @@ -0,0 +1,21 @@ +version: 2 + +macros: + - name: clean_up_pit + description: '{{ doc("clean_up_pit") }}' + arguments: + - name: snapshot_relation + type: string + description: > + The name of the dbt model that creates the snapshot table / view that has the logarithmic snapshot + logic applied. + - name: snapshot_trigger_column + type: string + description: > + The name of the boolean column inside the snapshot tables that activates/deactivates single snapshots. + If not set, the name defined inside the global variable 'datavault4dbt.snapshot_trigger_column' is used. + - name: sdts + type: string + description: > + The name of the snapshot date timestamp column inside the snapshot table. If not set, the name defined + inside the global variable 'datavault4dbt.sdts_alias' is used. diff --git a/macros/staging/stage.md b/macros/staging/stage.md new file mode 100644 index 00000000..41dfc314 --- /dev/null +++ b/macros/staging/stage.md @@ -0,0 +1,57 @@ +{% docs stage %} + +## Stage + +Creates the staging layer for the Data Vault model. This layer is mainly for hashing, and additionally gives the +option to create derived columns, conduct prejoins and add NULL values for missing columns. Always create one stage +per source table that you want to add to the Data Vault model. The staging layer is not to harmonize data — that +will be done in the later layers. + +### Usage + +```jinja +{{ datavault4dbt.stage( + source_model='source_account', + ldts='edwLoadDate', + rsrc='!SAP.Accounts', + hashed_columns={ + 'hk_account_h': ['account_number', 'account_key'], + 'hd_account_s': { + 'is_hashdiff': true, + 'columns': ['name', 'address', 'country', 'phone', 'email'] + } + }, + derived_columns={ + 'country_isocode': {'value': '!GER', 'datatype': 'STRING'} + } +) }} +``` + +### Multi-active stage + +```jinja +{{ datavault4dbt.stage( + source_model='source_contact_phones', + ldts='edwLoadDate', + rsrc='!CRM.ContactPhones', + hashed_columns={ + 'hk_contact_h': ['contact_id'], + 'hd_contact_phonenumber_s': { + 'is_hashdiff': true, + 'columns': ['phone_type', 'phone_number'] + } + }, + multi_active_config={ + 'multi_active_key': 'phone_type', + 'main_hashkey_column': 'hk_contact_h' + } +) }} +``` + +### Metadata block usage + +```jinja +{{ datavault4dbt.stage(yaml_metadata=meta) }} +``` + +{% enddocs %} diff --git a/macros/staging/stage.yml b/macros/staging/stage.yml new file mode 100644 index 00000000..b383addd --- /dev/null +++ b/macros/staging/stage.yml @@ -0,0 +1,82 @@ +version: 2 + +macros: + - name: stage + description: '{{ doc("stage") }}' + arguments: + - name: yaml_metadata + type: dictionary + description: > + Optional. A YAML metadata block that passes all parameters as a single dictionary instead + of specifying each one directly in the macro call. + - name: ldts + type: string + description: > + Name of the column inside the source data, that holds information about the Load Date Timestamp. + Can also be a SQL expression. + Examples: 'edwLoadDate' (column name as-is), + 'PARSE_TIMESTAMP('%Y-%m-%dT%H-%M-%S', edwLoadDate)' (applying a SQL function). + - name: rsrc + type: string + description: > + Name of the column inside the source data, that holds information about the Record Source. Can also + be a SQL expression or a static string. A static string must begin with a '!'. + Examples: 'edwRecordSource' (column), '!SAP.Accounts' (static string), + 'CONCAT(source_system, '||', source_object)' (expression). + - name: source_model + type: string | dictionary + description: > + Can be just a string holding the name of the referred dbt model to use as a source. But if the 'source' + functionality inside the .yml file is used, it must be a dictionary with 'source_name': 'source_table'. + Examples: 'source_account' (model name), {'source_data': 'source_account'} (dbt source reference). + - name: include_source_columns + type: boolean + description: > + Defines if all columns from the referred source table should be included in the result table, or if only + the added columns should be part of the result table. By default the source columns should be included. + - name: hashed_columns + type: dictionary + description: > + Defines the names and input for all hashkeys and hashdiffs to create. The key of each hash column is + the name of the hash column. The value for Hashkeys is a list of input Business Keys; for Hashdiffs + another dictionary with the pairs 'is_hashdiff:true' and 'columns: '. + Example: {'hk_account_h': ['account_number', 'account_key'], 'hd_account_s': {'is_hashdiff': true, 'columns': ['name', 'address']}}. + - name: derived_columns + type: dictionary + description: > + Defines values and datatypes for derived ('added' or 'calculated') columns. The values of this dictionary + are the desired column names; the value is another dictionary with the keys 'value' (holding a column name, + a SQL expression, or a static string beginning with '!') and 'datatype' (holding a valid SQL datatype). + Example: {'country_isocode': {'value': '!GER', 'datatype': 'STRING'}}. + - name: sequence + type: string + description: > + Name of the column inside the source data, that holds a sequence number generated during data source + extraction. Optional and not required. + Example: 'edwSequence'. + - name: prejoined_columns + type: dictionary + description: > + Defines information about columns that need to be prejoined. Most commonly used to create links when + the source data does not hold the Business Key but the technical key of the referred object. For each + prejoined column define: 'src_name', 'src_table' (or 'ref_model'), 'bk', 'this_column_name', + 'ref_column_name'. + - name: missing_columns + type: dictionary + description: > + If the schema of the source changes over time and columns are disappearing, this parameter gives you + the option to create additional columns holding NULL values. The dictionary holds the column names as + keys and the SQL datatypes as values. + Example: {'legacy_account_uuid': 'INT64', 'shipping_address': 'STRING'}. + - name: multi_active_config + type: dictionary + description: > + If the source data holds multi-active data, define here the column(s) holding the multi-active key and + the main hashkey column. The combination of multi-active key(s), the main hashkey and the ldts column + should be unique in the final result satellite. If not set, the stage will be treated as a single-active stage. + Example: {'multi_active_key': 'phonetype', 'main_hashkey_column': 'hk_contact_h'}. + - name: enable_ghost_records + type: boolean + description: > + If set to true, the stage will be created with ghost records. By default, ghost records are enabled. + Optional Parameter. diff --git a/macros/tables/bigquery/control_snap_v0.yml b/macros/tables/bigquery/control_snap_v0.yml new file mode 100644 index 00000000..c82e7e79 --- /dev/null +++ b/macros/tables/bigquery/control_snap_v0.yml @@ -0,0 +1,21 @@ +version: 2 + +macros: + - name: default__control_snap_v0 + description: > + Default implementation of the control_snap_v0 macro, co-located in the bigquery adapter folder as BigQuery + uses this fallback via adapter.dispatch(). See the parent control_snap_v0 macro for full parameter + documentation. + arguments: + - name: start_date + type: timestamp + description: Earliest timestamp for the snapshot table. Time part must be '00:00:00'. + - name: daily_snapshot_time + type: time + description: Time that daily snapshots should have (e.g. '07:00:00'). + - name: sdts_alias + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: end_date + type: timestamp + description: Latest timestamp for the snapshot table. Optional. diff --git a/macros/tables/bigquery/control_snap_v1.yml b/macros/tables/bigquery/control_snap_v1.yml new file mode 100644 index 00000000..92b5af54 --- /dev/null +++ b/macros/tables/bigquery/control_snap_v1.yml @@ -0,0 +1,20 @@ +version: 2 + +macros: + - name: bigquery__control_snap_v1 + description: > + BigQuery-specific adapter implementation of the control_snap_v1 macro. Called automatically via + adapter.dispatch() when running on BigQuery. See the parent control_snap_v1 macro for full parameter + documentation. + arguments: + - name: control_snap_v0 + type: string + description: The name of the underlying version 0 control snapshot table. + - name: log_logic + type: dictionary + description: > + Defining the desired durations of each granularity ('daily', 'weekly', 'monthly', 'yearly'). + Optional — if not set, all snapshots remain active. + - name: sdts_alias + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. diff --git a/macros/tables/bigquery/eff_sat_v0.yml b/macros/tables/bigquery/eff_sat_v0.yml new file mode 100644 index 00000000..fa0f6cf4 --- /dev/null +++ b/macros/tables/bigquery/eff_sat_v0.yml @@ -0,0 +1,32 @@ +version: 2 + +macros: + - name: default__eff_sat_v0 + description: > + Default implementation of the eff_sat_v0 macro, co-located in the bigquery adapter folder as BigQuery uses + this fallback via adapter.dispatch(). See the parent eff_sat_v0 macro for full parameter documentation. + arguments: + - name: tracked_hashkey + type: string + description: Name of the hashkey column to be tracked. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: is_active_alias + type: string + description: Name of the active flag column. Optional, defaults to 'datavault4dbt.is_active_alias'. + - name: source_model + type: string + description: Name of the source model. + - name: source_is_single_batch + type: boolean + description: REQUIRED. Whether the source contains only one batch. Must be explicitly set. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be disabled. Optional. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/bigquery/hub.yml b/macros/tables/bigquery/hub.yml new file mode 100644 index 00000000..64c49168 --- /dev/null +++ b/macros/tables/bigquery/hub.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: default__hub + description: > + Default implementation of the hub macro, co-located in the bigquery adapter folder as BigQuery uses + this fallback via adapter.dispatch(). See the parent hub macro for full parameter documentation. + arguments: + - name: hashkey + type: string + description: Name of the hashkey column inside the stage, used as PK of the Hub. + - name: business_keys + type: string | list of strings + description: Name(s) of the business key columns loaded into the hub. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include in the Hub. Optional, defaults to empty list. diff --git a/macros/tables/bigquery/link.yml b/macros/tables/bigquery/link.yml new file mode 100644 index 00000000..6aa0db32 --- /dev/null +++ b/macros/tables/bigquery/link.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: default__link + description: > + Default implementation of the link macro, co-located in the bigquery adapter folder as BigQuery uses + this fallback via adapter.dispatch(). See the parent link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: Name of the link hashkey column inside the stage. + - name: foreign_hashkeys + type: list of strings + description: List of all hashkey columns inside the link that refer to other hub entities. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include in the Link. Optional, defaults to empty list. diff --git a/macros/tables/bigquery/ma_sat_v0.yml b/macros/tables/bigquery/ma_sat_v0.yml new file mode 100644 index 00000000..ad1164bc --- /dev/null +++ b/macros/tables/bigquery/ma_sat_v0.yml @@ -0,0 +1,39 @@ +version: 2 + +macros: + - name: default__ma_sat_v0 + description: > + Default implementation of the ma_sat_v0 macro, co-located in the bigquery adapter folder as BigQuery uses + this fallback via adapter.dispatch(). Uses QUALIFY for intra-batch deduplication. + See the parent ma_sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this satellite. + - name: src_ma_key + type: string | list of strings + description: Name(s) of the multi-active key column(s). + - name: src_payload + type: list of strings + description: List of descriptive attribute columns (excluding the multi-active key). + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Skips QUALIFY deduplication. Optional, default False. diff --git a/macros/tables/bigquery/ma_sat_v1.yml b/macros/tables/bigquery/ma_sat_v1.yml new file mode 100644 index 00000000..913b4201 --- /dev/null +++ b/macros/tables/bigquery/ma_sat_v1.yml @@ -0,0 +1,32 @@ +version: 2 + +macros: + - name: default__ma_sat_v1 + description: > + Default implementation of the ma_sat_v1 macro, co-located in the bigquery adapter folder as BigQuery uses + this fallback via adapter.dispatch(). See the parent ma_sat_v1 macro for full parameter documentation. + arguments: + - name: sat_v0 + type: string + description: Name of the underlying version 0 multi-active satellite. + - name: hashkey + type: string + description: Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: Name of the hashdiff column inside the version 0 satellite. + - name: ma_attribute + type: string | list of strings + description: Name of the multi-active attribute(s) in the v0 satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag derived from the ledts column. Optional, default False. diff --git a/macros/tables/bigquery/nh_link.yml b/macros/tables/bigquery/nh_link.yml new file mode 100644 index 00000000..ba48d690 --- /dev/null +++ b/macros/tables/bigquery/nh_link.yml @@ -0,0 +1,38 @@ +version: 2 + +macros: + - name: default__nh_link + description: > + Default implementation of the nh_link macro, co-located in the bigquery adapter folder as BigQuery uses + this fallback via adapter.dispatch(). See the parent nh_link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: Name of the non-historized link hashkey column inside the stage. + - name: payload + type: list of strings + description: List of descriptive attribute columns forming the payload. + - name: foreign_hashkeys + type: list of strings + description: List of all hashkey columns inside the link that refer to other hub entities. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: union_strategy + type: "'all' | 'distinct'" + description: Defines how multiple sources should be unioned. Default 'all'. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/bigquery/nh_sat.yml b/macros/tables/bigquery/nh_sat.yml new file mode 100644 index 00000000..8389268c --- /dev/null +++ b/macros/tables/bigquery/nh_sat.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: default__nh_sat + description: > + Default implementation of the nh_sat macro, co-located in the bigquery adapter folder as BigQuery uses + this fallback via adapter.dispatch(). See the parent nh_sat macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns for this non-historized satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/bigquery/pit.yml b/macros/tables/bigquery/pit.yml new file mode 100644 index 00000000..afd890d2 --- /dev/null +++ b/macros/tables/bigquery/pit.yml @@ -0,0 +1,47 @@ +version: 2 + +macros: + - name: default__pit + description: > + Default implementation of the pit macro, co-located in the bigquery adapter folder as BigQuery uses + this fallback via adapter.dispatch(). See the parent pit macro for full parameter documentation. + arguments: + - name: pit_type + type: string + description: String to insert into the 'pit_type' column, prefixed by !. Optional. + - name: tracked_entity + type: string + description: Name of the tracked Hub entity. + - name: hashkey + type: string + description: Name of the hashkey column inside the tracked Hub. + - name: sat_names + type: list of strings + description: List of all satellite names to include in this PIT table. + - name: ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: sdts + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: custom_rsrc + type: string + description: Custom string to insert into the rsrc column. Optional. + - name: ledts + type: string + description: Name of the load-end-date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: snapshot_relation + type: string + description: Name of the snapshot relation model. + - name: snapshot_trigger_column + type: string + description: Name of the boolean column that identifies active snapshots. + - name: dimension_key + type: string + description: Desired name of the dimension key inside the PIT table. + - name: refer_to_ghost_records + type: boolean + description: Whether a NULL satellite hashkey should be replaced by the unknown key. Optional, default True. + - name: snapshot_optimization + type: boolean + description: Not applicable for this default implementation. Optional, default False. diff --git a/macros/tables/bigquery/rec_track_sat.yml b/macros/tables/bigquery/rec_track_sat.yml new file mode 100644 index 00000000..a12fb666 --- /dev/null +++ b/macros/tables/bigquery/rec_track_sat.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: default__rec_track_sat + description: > + Default implementation of the rec_track_sat macro, co-located in the bigquery adapter folder as BigQuery + uses this fallback via adapter.dispatch(). See the parent rec_track_sat macro for full parameter + documentation. + arguments: + - name: tracked_hashkey + type: string + description: The name of the hashkey column you want to track. + - name: source_models + type: dictionary + description: Dictionary with source model information, including 'rsrc_static' and optionally 'hk_column'. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: src_stg + type: string + description: Name of the source stage model. Optional, defaults to 'datavault4dbt.stg_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/bigquery/ref_hub.yml b/macros/tables/bigquery/ref_hub.yml new file mode 100644 index 00000000..7ef9b9c2 --- /dev/null +++ b/macros/tables/bigquery/ref_hub.yml @@ -0,0 +1,23 @@ +version: 2 + +macros: + - name: default__ref_hub + description: > + Default implementation of the ref_hub macro, co-located in the bigquery adapter folder as BigQuery uses + this fallback via adapter.dispatch(). See the parent ref_hub macro for full parameter documentation. + arguments: + - name: ref_keys + type: string | list of strings + description: Name of the reference key(s) available in the source model(s). + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/bigquery/ref_sat_v0.yml b/macros/tables/bigquery/ref_sat_v0.yml new file mode 100644 index 00000000..82ce479a --- /dev/null +++ b/macros/tables/bigquery/ref_sat_v0.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: default__ref_sat_v0 + description: > + Default implementation of the ref_sat_v0 macro, co-located in the bigquery adapter folder as BigQuery uses + this fallback via adapter.dispatch(). Uses QUALIFY for deduplication. + See the parent ref_sat_v0 macro for full parameter documentation. + arguments: + - name: parent_ref_keys + type: string | list of strings + description: Name of the reference key(s) of the parent ref_hub. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this reference satellite. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/bigquery/ref_sat_v1.yml b/macros/tables/bigquery/ref_sat_v1.yml new file mode 100644 index 00000000..c603841d --- /dev/null +++ b/macros/tables/bigquery/ref_sat_v1.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: default__ref_sat_v1 + description: > + Default implementation of the ref_sat_v1 macro, co-located in the bigquery adapter folder as BigQuery uses + this fallback via adapter.dispatch(). See the parent ref_sat_v1 macro for full parameter documentation. + arguments: + - name: ref_sat_v0 + type: string + description: Name of the underlying ref_sat_v0 dbt model. + - name: ref_keys + type: string | list of strings + description: Name(s) of the reference key(s) in the underlying reference sat v0. + - name: hashdiff + type: string + description: Name of the Hashdiff column in the underlying reference sat v0. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag. Optional, default False. diff --git a/macros/tables/bigquery/ref_table.yml b/macros/tables/bigquery/ref_table.yml new file mode 100644 index 00000000..cbf20e57 --- /dev/null +++ b/macros/tables/bigquery/ref_table.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: default__ref_table + description: > + Default implementation of the ref_table macro, co-located in the bigquery adapter folder as BigQuery uses + this fallback via adapter.dispatch(). See the parent ref_table macro for full parameter documentation. + arguments: + - name: ref_hub + type: string + description: Name of the underlying ref_hub model. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ref_satellites + type: string | list of strings + description: Name(s) of the reference satellites to include. + - name: historized + type: string + description: Possible values are 'full', 'latest', or 'snapshot'. + - name: snapshot_relation + type: string + description: Only required if historized is 'snapshot'. Name of the snapshot_v1 model. + - name: snapshot_trigger_column + type: string + description: Only required if historized is 'snapshot'. Trigger column name in the snapshot relation. diff --git a/macros/tables/bigquery/sat_v0.yml b/macros/tables/bigquery/sat_v0.yml new file mode 100644 index 00000000..92d9323f --- /dev/null +++ b/macros/tables/bigquery/sat_v0.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: default__sat_v0 + description: > + Default implementation of the sat_v0 macro, co-located in the bigquery adapter folder as BigQuery uses + this fallback via adapter.dispatch(). Uses QUALIFY for intra-batch deduplication. + See the parent sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this satellite. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns that form the satellite payload. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Skips QUALIFY deduplication. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/bigquery/sat_v1.yml b/macros/tables/bigquery/sat_v1.yml new file mode 100644 index 00000000..200ca731 --- /dev/null +++ b/macros/tables/bigquery/sat_v1.yml @@ -0,0 +1,32 @@ +version: 2 + +macros: + - name: default__sat_v1 + description: > + Default implementation of the sat_v1 macro, co-located in the bigquery adapter folder as BigQuery uses + this fallback via adapter.dispatch(). See the parent sat_v1 macro for full parameter documentation. + arguments: + - name: sat_v0 + type: string + description: Name of the underlying version 0 satellite. + - name: hashkey + type: string + description: Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: Name of the hashdiff column inside the version 0 satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag derived from the ledts column. Optional, default False. + - name: include_payload + type: boolean + description: Whether to include the payload columns from sat v0. Optional, default True. diff --git a/macros/tables/control_snap_v0.md b/macros/tables/control_snap_v0.md new file mode 100644 index 00000000..c10053ef --- /dev/null +++ b/macros/tables/control_snap_v0.md @@ -0,0 +1,30 @@ +{% docs control_snap_v0 %} + +## Control Snapshot v0 + +Creates a snapshot table to control snapshot-based tables like PITs and Bridges. The snapshot table will hold +daily snapshots starting at a specific start_date with a configurable daytime. Usually one snapshot table per +Data Vault environment is created. The model needs to be scheduled daily at the time matching the desired +snapshot time. + +In addition to the actual snapshot datetimestamp (sdts), the macro generates the following metadata columns: +`replacement_sdts`, `caption`, `is_hourly`, `is_daily`, `is_beginning_of_week`, `is_end_of_week`, +`is_beginning_of_month`, `is_end_of_month`, `is_beginning_of_quarter`, `is_end_of_quarter`, +`is_beginning_of_year`, `is_end_of_year`, `comment`, `force_active`. + +### Usage + +```jinja +{{ datavault4dbt.control_snap_v0( + start_date='2020-01-01T00-00-00', + daily_snapshot_time='07:00:00' +) }} +``` + +### Metadata block usage + +```jinja +{{ datavault4dbt.control_snap_v0(yaml_metadata=meta) }} +``` + +{% enddocs %} diff --git a/macros/tables/control_snap_v0.yml b/macros/tables/control_snap_v0.yml new file mode 100644 index 00000000..ede8f90b --- /dev/null +++ b/macros/tables/control_snap_v0.yml @@ -0,0 +1,33 @@ +version: 2 + +macros: + - name: control_snap_v0 + description: '{{ doc("control_snap_v0") }}' + arguments: + - name: yaml_metadata + type: dictionary + description: > + Optional. A YAML metadata block that passes all parameters as a single dictionary instead + of specifying each one directly in the macro call. + - name: start_date + type: timestamp + description: > + Defines the earliest timestamp that should be available inside the snapshot_table. The time part of this + timestamp needs to be set to '00:00:00'. The format of this timestamp must equal to the timestamp format + defined in the global variable 'datavault4dbt.timestamp_format'. + Examples: '2015-01-01T00-00-00' — This snapshot table would hold daily snapshots beginning at 2015. + - name: daily_snapshot_time + type: time + description: > + Defines the time that your daily snapshots should have. Usually this is either something right before + daily business starts, or after daily business is over. + Examples: '07:30:00', '23:00:00'. + - name: sdts_alias + type: string + description: > + Defines the name of the snapshot date timestamp column inside the snapshot_table. It is optional, + if not set will use the global variable 'datavault4dbt.sdts_alias' set inside dbt_project.yml. + - name: end_date + type: timestamp + description: > + Defines the latest timestamp that should be available inside the snapshot_table. diff --git a/macros/tables/control_snap_v1.md b/macros/tables/control_snap_v1.md new file mode 100644 index 00000000..f2d891c3 --- /dev/null +++ b/macros/tables/control_snap_v1.md @@ -0,0 +1,39 @@ +{% docs control_snap_v1 %} + +## Control Snapshot v1 + +Creates a view that extends an existing control_snap_v0 table by dynamically applying logarithmic snapshot logic. +The further you look into the past, the more coarsely the snapshots are granulated — for example, keeping daily +snapshots for the past 30 days but only weekly snapshots for the past 6 months and monthly snapshots for the +past 3 years. + +This procedure strongly reduces the number of active snapshots and therefore the computation inside all PITs +and Bridges. + +In addition to the logarithmic `is_active` column, the following dynamic columns are generated: +`is_latest`, `is_current_year`, `is_last_year`, `is_rolling_year`, `is_last_rolling_year`. + +Note: Whenever a logarithmic snapshot logic is used with PIT tables, the `clean_up_pit` post-hook must be +applied to each PIT table to remove inactive records. + +### Usage + +```jinja +{{ datavault4dbt.control_snap_v1( + control_snap_v0='control_snap_v0', + log_logic={ + 'daily': {'duration': 3, 'unit': 'MONTH'}, + 'weekly': {'duration': 1, 'unit': 'YEAR'}, + 'monthly': {'duration': 5, 'unit': 'YEAR'}, + 'yearly': {'forever': 'TRUE'} + } +) }} +``` + +### Metadata block usage + +```jinja +{{ datavault4dbt.control_snap_v1(yaml_metadata=meta) }} +``` + +{% enddocs %} diff --git a/macros/tables/control_snap_v1.yml b/macros/tables/control_snap_v1.yml new file mode 100644 index 00000000..04bde654 --- /dev/null +++ b/macros/tables/control_snap_v1.yml @@ -0,0 +1,31 @@ +version: 2 + +macros: + - name: control_snap_v1 + description: '{{ doc("control_snap_v1") }}' + arguments: + - name: yaml_metadata + type: dictionary + description: > + Optional. A YAML metadata block that passes all parameters as a single dictionary instead + of specifying each one directly in the macro call. + - name: control_snap_v0 + type: string + description: > + The name of the underlying version 0 control snapshot table. Needs to be available as a dbt model. + - name: log_logic + type: dictionary + description: > + Defining the desired durations of each granularity. Available granularities are 'daily', 'weekly', + 'monthly', and 'yearly'. For each granularity the duration can be defined as an integer and the time + unit for that duration (e.g. DAY, WEEK, MONTH, QUARTER, YEAR in BigQuery). A granularity can also be + set to 'forever'. If log_logic is not set, no logic will be applied and all snapshots will stay active. + The duration is always counted from the current date. + EXASOL: Due to a missing 'DAY OF WEEK' function, 'weekly' is not supported and must be left out. + Example: {'daily': {'duration': 3, 'unit': 'MONTH'}, 'weekly': {'duration': 1, 'unit': 'YEAR'}, + 'monthly': {'duration': 5, 'unit': 'YEAR'}, 'yearly': {'forever': 'TRUE'}}. + - name: sdts_alias + type: string + description: > + Defines the name of the snapshot date timestamp column inside the snapshot_table. It is optional, + if not set will use the global variable 'datavault4dbt.sdts_alias' set inside dbt_project.yml. diff --git a/macros/tables/databricks/control_snap_v0.yml b/macros/tables/databricks/control_snap_v0.yml new file mode 100644 index 00000000..365d3f51 --- /dev/null +++ b/macros/tables/databricks/control_snap_v0.yml @@ -0,0 +1,21 @@ +version: 2 + +macros: + - name: databricks__control_snap_v0 + description: > + Databricks-specific adapter implementation of the control_snap_v0 macro. Called automatically via + adapter.dispatch() when running on Databricks. See the parent control_snap_v0 macro for full parameter + documentation. + arguments: + - name: start_date + type: timestamp + description: Earliest timestamp for the snapshot table. Time part must be '00:00:00'. + - name: daily_snapshot_time + type: time + description: Time that daily snapshots should have (e.g. '07:00:00'). + - name: sdts_alias + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: end_date + type: timestamp + description: Latest timestamp for the snapshot table. Optional. diff --git a/macros/tables/databricks/control_snap_v1.yml b/macros/tables/databricks/control_snap_v1.yml new file mode 100644 index 00000000..34d1fc30 --- /dev/null +++ b/macros/tables/databricks/control_snap_v1.yml @@ -0,0 +1,20 @@ +version: 2 + +macros: + - name: databricks__control_snap_v1 + description: > + Databricks-specific adapter implementation of the control_snap_v1 macro. Called automatically via + adapter.dispatch() when running on Databricks. See the parent control_snap_v1 macro for full parameter + documentation. + arguments: + - name: control_snap_v0 + type: string + description: The name of the underlying version 0 control snapshot table. + - name: log_logic + type: dictionary + description: > + Defining the desired durations of each granularity ('daily', 'weekly', 'monthly', 'yearly'). + Optional — if not set, all snapshots remain active. + - name: sdts_alias + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. diff --git a/macros/tables/databricks/eff_sat_v0.yml b/macros/tables/databricks/eff_sat_v0.yml new file mode 100644 index 00000000..6194d743 --- /dev/null +++ b/macros/tables/databricks/eff_sat_v0.yml @@ -0,0 +1,32 @@ +version: 2 + +macros: + - name: databricks__eff_sat_v0 + description: > + Databricks-specific adapter implementation of the eff_sat_v0 macro. Called automatically via adapter.dispatch() + when running on Databricks. See the parent eff_sat_v0 macro for full parameter documentation. + arguments: + - name: tracked_hashkey + type: string + description: Name of the hashkey column to be tracked. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: is_active_alias + type: string + description: Name of the active flag column. Optional, defaults to 'datavault4dbt.is_active_alias'. + - name: source_model + type: string + description: Name of the source model. + - name: source_is_single_batch + type: boolean + description: REQUIRED. Whether the source contains only one batch. Must be explicitly set. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be disabled. Optional. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/databricks/hub.yml b/macros/tables/databricks/hub.yml new file mode 100644 index 00000000..f26d8b01 --- /dev/null +++ b/macros/tables/databricks/hub.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: databricks__hub + description: > + Databricks-specific adapter implementation of the hub macro. Called automatically via adapter.dispatch() + when running on Databricks. See the parent hub macro for full parameter documentation. + arguments: + - name: hashkey + type: string + description: Name of the hashkey column inside the stage, used as PK of the Hub. + - name: business_keys + type: string | list of strings + description: Name(s) of the business key columns loaded into the hub. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include in the Hub. Optional, defaults to empty list. diff --git a/macros/tables/databricks/link.yml b/macros/tables/databricks/link.yml new file mode 100644 index 00000000..dc37ca14 --- /dev/null +++ b/macros/tables/databricks/link.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: databricks__link + description: > + Databricks-specific adapter implementation of the link macro. Called automatically via adapter.dispatch() + when running on Databricks. See the parent link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: Name of the link hashkey column inside the stage. + - name: foreign_hashkeys + type: list of strings + description: List of all hashkey columns inside the link that refer to other hub entities. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include in the Link. Optional, defaults to empty list. diff --git a/macros/tables/databricks/ma_sat_v0.yml b/macros/tables/databricks/ma_sat_v0.yml new file mode 100644 index 00000000..b196e823 --- /dev/null +++ b/macros/tables/databricks/ma_sat_v0.yml @@ -0,0 +1,39 @@ +version: 2 + +macros: + - name: databricks__ma_sat_v0 + description: > + Databricks-specific adapter implementation of the ma_sat_v0 macro. Called automatically via adapter.dispatch() + when running on Databricks. Uses QUALIFY for intra-batch deduplication. + See the parent ma_sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this satellite. + - name: src_ma_key + type: string | list of strings + description: Name(s) of the multi-active key column(s). + - name: src_payload + type: list of strings + description: List of descriptive attribute columns (excluding the multi-active key). + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Skips QUALIFY deduplication. Optional, default False. diff --git a/macros/tables/databricks/ma_sat_v1.yml b/macros/tables/databricks/ma_sat_v1.yml new file mode 100644 index 00000000..6f4f0271 --- /dev/null +++ b/macros/tables/databricks/ma_sat_v1.yml @@ -0,0 +1,32 @@ +version: 2 + +macros: + - name: databricks__ma_sat_v1 + description: > + Databricks-specific adapter implementation of the ma_sat_v1 macro. Called automatically via adapter.dispatch() + when running on Databricks. See the parent ma_sat_v1 macro for full parameter documentation. + arguments: + - name: sat_v0 + type: string + description: Name of the underlying version 0 multi-active satellite. + - name: hashkey + type: string + description: Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: Name of the hashdiff column inside the version 0 satellite. + - name: ma_attribute + type: string | list of strings + description: Name of the multi-active attribute(s) in the v0 satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag derived from the ledts column. Optional, default False. diff --git a/macros/tables/databricks/nh_link.yml b/macros/tables/databricks/nh_link.yml new file mode 100644 index 00000000..06af4c12 --- /dev/null +++ b/macros/tables/databricks/nh_link.yml @@ -0,0 +1,38 @@ +version: 2 + +macros: + - name: databricks__nh_link + description: > + Databricks-specific adapter implementation of the nh_link macro. Called automatically via adapter.dispatch() + when running on Databricks. See the parent nh_link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: Name of the non-historized link hashkey column inside the stage. + - name: payload + type: list of strings + description: List of descriptive attribute columns forming the payload. + - name: foreign_hashkeys + type: list of strings + description: List of all hashkey columns inside the link that refer to other hub entities. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: union_strategy + type: "'all' | 'distinct'" + description: Defines how multiple sources should be unioned. Default 'all'. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/databricks/nh_sat.yml b/macros/tables/databricks/nh_sat.yml new file mode 100644 index 00000000..6a888745 --- /dev/null +++ b/macros/tables/databricks/nh_sat.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: databricks__nh_sat + description: > + Databricks-specific adapter implementation of the nh_sat macro. Called automatically via adapter.dispatch() + when running on Databricks. See the parent nh_sat macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns for this non-historized satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/databricks/pit.yml b/macros/tables/databricks/pit.yml new file mode 100644 index 00000000..2c1126d0 --- /dev/null +++ b/macros/tables/databricks/pit.yml @@ -0,0 +1,47 @@ +version: 2 + +macros: + - name: databricks__pit + description: > + Databricks-specific adapter implementation of the pit macro. Called automatically via adapter.dispatch() + when running on Databricks. See the parent pit macro for full parameter documentation. + arguments: + - name: pit_type + type: string + description: String to insert into the 'pit_type' column, prefixed by !. Optional. + - name: tracked_entity + type: string + description: Name of the tracked Hub entity. + - name: hashkey + type: string + description: Name of the hashkey column inside the tracked Hub. + - name: sat_names + type: list of strings + description: List of all satellite names to include in this PIT table. + - name: ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: sdts + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: custom_rsrc + type: string + description: Custom string to insert into the rsrc column. Optional. + - name: ledts + type: string + description: Name of the load-end-date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: snapshot_relation + type: string + description: Name of the snapshot relation model. + - name: snapshot_trigger_column + type: string + description: Name of the boolean column that identifies active snapshots. + - name: dimension_key + type: string + description: Desired name of the dimension key inside the PIT table. + - name: refer_to_ghost_records + type: boolean + description: Whether a NULL satellite hashkey should be replaced by the unknown key. Optional, default True. + - name: snapshot_optimization + type: boolean + description: Whether to apply snapshot optimization logic. Optional, default False. diff --git a/macros/tables/databricks/rec_track_sat.yml b/macros/tables/databricks/rec_track_sat.yml new file mode 100644 index 00000000..ec3802e0 --- /dev/null +++ b/macros/tables/databricks/rec_track_sat.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: databricks__rec_track_sat + description: > + Databricks-specific adapter implementation of the rec_track_sat macro. Called automatically via + adapter.dispatch() when running on Databricks. See the parent rec_track_sat macro for full parameter + documentation. + arguments: + - name: tracked_hashkey + type: string + description: The name of the hashkey column you want to track. + - name: source_models + type: dictionary + description: Dictionary with source model information, including 'rsrc_static' and optionally 'hk_column'. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: src_stg + type: string + description: Name of the source stage model. Optional, defaults to 'datavault4dbt.stg_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/databricks/ref_hub.yml b/macros/tables/databricks/ref_hub.yml new file mode 100644 index 00000000..1173171e --- /dev/null +++ b/macros/tables/databricks/ref_hub.yml @@ -0,0 +1,23 @@ +version: 2 + +macros: + - name: databricks__ref_hub + description: > + Databricks-specific adapter implementation of the ref_hub macro. Called automatically via adapter.dispatch() + when running on Databricks. See the parent ref_hub macro for full parameter documentation. + arguments: + - name: ref_keys + type: string | list of strings + description: Name of the reference key(s) available in the source model(s). + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/databricks/ref_sat_v0.yml b/macros/tables/databricks/ref_sat_v0.yml new file mode 100644 index 00000000..e2d26299 --- /dev/null +++ b/macros/tables/databricks/ref_sat_v0.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: databricks__ref_sat_v0 + description: > + Databricks-specific adapter implementation of the ref_sat_v0 macro. Called automatically via adapter.dispatch() + when running on Databricks. Uses QUALIFY for deduplication. + See the parent ref_sat_v0 macro for full parameter documentation. + arguments: + - name: parent_ref_keys + type: string | list of strings + description: Name of the reference key(s) of the parent ref_hub. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this reference satellite. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/databricks/ref_sat_v1.yml b/macros/tables/databricks/ref_sat_v1.yml new file mode 100644 index 00000000..92360566 --- /dev/null +++ b/macros/tables/databricks/ref_sat_v1.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: databricks__ref_sat_v1 + description: > + Databricks-specific adapter implementation of the ref_sat_v1 macro. Called automatically via adapter.dispatch() + when running on Databricks. See the parent ref_sat_v1 macro for full parameter documentation. + arguments: + - name: ref_sat_v0 + type: string + description: Name of the underlying ref_sat_v0 dbt model. + - name: ref_keys + type: string | list of strings + description: Name(s) of the reference key(s) in the underlying reference sat v0. + - name: hashdiff + type: string + description: Name of the Hashdiff column in the underlying reference sat v0. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag. Optional, default False. diff --git a/macros/tables/databricks/ref_table.yml b/macros/tables/databricks/ref_table.yml new file mode 100644 index 00000000..3dc00fae --- /dev/null +++ b/macros/tables/databricks/ref_table.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: databricks__ref_table + description: > + Databricks-specific adapter implementation of the ref_table macro. Called automatically via adapter.dispatch() + when running on Databricks. See the parent ref_table macro for full parameter documentation. + arguments: + - name: ref_hub + type: string + description: Name of the underlying ref_hub model. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ref_satellites + type: string | list of strings + description: Name(s) of the reference satellites to include. + - name: historized + type: string + description: Possible values are 'full', 'latest', or 'snapshot'. + - name: snapshot_relation + type: string + description: Only required if historized is 'snapshot'. Name of the snapshot_v1 model. + - name: snapshot_trigger_column + type: string + description: Only required if historized is 'snapshot'. Trigger column name in the snapshot relation. diff --git a/macros/tables/databricks/sat_v0.yml b/macros/tables/databricks/sat_v0.yml new file mode 100644 index 00000000..65aabe43 --- /dev/null +++ b/macros/tables/databricks/sat_v0.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: databricks__sat_v0 + description: > + Databricks-specific adapter implementation of the sat_v0 macro. Called automatically via adapter.dispatch() + when running on Databricks. Uses QUALIFY for intra-batch deduplication. + See the parent sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this satellite. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns that form the satellite payload. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Skips QUALIFY deduplication. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/databricks/sat_v1.yml b/macros/tables/databricks/sat_v1.yml new file mode 100644 index 00000000..c4b88c04 --- /dev/null +++ b/macros/tables/databricks/sat_v1.yml @@ -0,0 +1,32 @@ +version: 2 + +macros: + - name: databricks__sat_v1 + description: > + Databricks-specific adapter implementation of the sat_v1 macro. Called automatically via adapter.dispatch() + when running on Databricks. See the parent sat_v1 macro for full parameter documentation. + arguments: + - name: sat_v0 + type: string + description: Name of the underlying version 0 satellite. + - name: hashkey + type: string + description: Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: Name of the hashdiff column inside the version 0 satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag derived from the ledts column. Optional, default False. + - name: include_payload + type: boolean + description: Whether to include the payload columns from sat v0. Optional, default True. diff --git a/macros/tables/eff_sat_v0.md b/macros/tables/eff_sat_v0.md new file mode 100644 index 00000000..530aa61d --- /dev/null +++ b/macros/tables/eff_sat_v0.md @@ -0,0 +1,38 @@ +{% docs eff_sat_v0 %} + +## Effectivity Satellite v0 + +Creates an Effectivity Satellite that tracks the active/inactive status of a link relationship over time. +An effectivity satellite contains a boolean flag ('is_active') that indicates whether a given relationship +was active at each load timestamp. + +Note: `source_is_single_batch` is a required parameter — it must be explicitly set to `true` or `false`. + +### Usage + +```jinja +{{ datavault4dbt.eff_sat_v0( + source_model='stg_account_contact', + tracked_hashkey='hk_account_contact_l', + source_is_single_batch=false +) }} +``` + +### With custom alias for the active flag + +```jinja +{{ datavault4dbt.eff_sat_v0( + source_model='stg_account_contact', + tracked_hashkey='hk_account_contact_l', + is_active_alias='is_active', + source_is_single_batch=false +) }} +``` + +### Metadata block usage + +```jinja +{{ datavault4dbt.eff_sat_v0(yaml_metadata=meta) }} +``` + +{% enddocs %} diff --git a/macros/tables/eff_sat_v0.yml b/macros/tables/eff_sat_v0.yml new file mode 100644 index 00000000..94b8fec3 --- /dev/null +++ b/macros/tables/eff_sat_v0.yml @@ -0,0 +1,44 @@ +version: 2 + +macros: + - name: eff_sat_v0 + description: '{{ doc("eff_sat_v0") }}' + arguments: + - name: yaml_metadata + type: dictionary + description: > + Optional. A YAML metadata block that passes all parameters as a single dictionary instead + of specifying each one directly in the macro call. + - name: source_model + type: string + description: > + Name of the source model. + - name: tracked_hashkey + type: string + description: > + Name of the hashkey column to be tracked. + - name: src_ldts + type: string + description: > + Name of the loaddate column in the source model. Optional. + - name: src_rsrc + type: string + description: > + Name of the record source column in the source model. Optional. + - name: is_active_alias + type: string + description: > + Name of the new active flag column. Optional. + - name: source_is_single_batch + type: boolean + description: > + REQUIRED. Whether the source contains only one batch. Must be defined explicitly (true/false). + - name: disable_hwm + type: boolean + description: > + Whether the High Water Mark should be disabled or not. Optional. + - name: additional_columns + type: list | string + description: > + Additional columns from source system or derived columns which should be part of the eff_sat. + The columns need to be available in all source models. Optional parameter, defaults to empty list. diff --git a/macros/tables/exasol/control_snap_v0.yml b/macros/tables/exasol/control_snap_v0.yml new file mode 100644 index 00000000..7243cb3f --- /dev/null +++ b/macros/tables/exasol/control_snap_v0.yml @@ -0,0 +1,21 @@ +version: 2 + +macros: + - name: exasol__control_snap_v0 + description: > + Exasol-specific adapter implementation of the control_snap_v0 macro. Called automatically via + adapter.dispatch() when running on Exasol. See the parent control_snap_v0 macro for full parameter + documentation. + arguments: + - name: start_date + type: timestamp + description: Earliest timestamp for the snapshot table. Time part must be '00:00:00'. + - name: daily_snapshot_time + type: time + description: Time that daily snapshots should have (e.g. '07:00:00'). + - name: sdts_alias + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: end_date + type: timestamp + description: Latest timestamp for the snapshot table. Optional. diff --git a/macros/tables/exasol/control_snap_v1.yml b/macros/tables/exasol/control_snap_v1.yml new file mode 100644 index 00000000..68d85fce --- /dev/null +++ b/macros/tables/exasol/control_snap_v1.yml @@ -0,0 +1,20 @@ +version: 2 + +macros: + - name: exasol__control_snap_v1 + description: > + Exasol-specific adapter implementation of the control_snap_v1 macro. Called automatically via + adapter.dispatch() when running on Exasol. See the parent control_snap_v1 macro for full parameter + documentation. + arguments: + - name: control_snap_v0 + type: string + description: The name of the underlying version 0 control snapshot table. + - name: log_logic + type: dictionary + description: > + Defining the desired durations of each granularity ('daily', 'weekly', 'monthly', 'yearly'). + Optional — if not set, all snapshots remain active. + - name: sdts_alias + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. diff --git a/macros/tables/exasol/eff_sat_v0.yml b/macros/tables/exasol/eff_sat_v0.yml new file mode 100644 index 00000000..e5caec86 --- /dev/null +++ b/macros/tables/exasol/eff_sat_v0.yml @@ -0,0 +1,32 @@ +version: 2 + +macros: + - name: exasol__eff_sat_v0 + description: > + Exasol-specific adapter implementation of the eff_sat_v0 macro. Called automatically via adapter.dispatch() + when running on Exasol. See the parent eff_sat_v0 macro for full parameter documentation. + arguments: + - name: tracked_hashkey + type: string + description: Name of the hashkey column to be tracked. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: is_active_alias + type: string + description: Name of the active flag column. Optional, defaults to 'datavault4dbt.is_active_alias'. + - name: source_model + type: string + description: Name of the source model. + - name: source_is_single_batch + type: boolean + description: REQUIRED. Whether the source contains only one batch. Must be explicitly set. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be disabled. Optional. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/exasol/hub.yml b/macros/tables/exasol/hub.yml new file mode 100644 index 00000000..66e6c3a6 --- /dev/null +++ b/macros/tables/exasol/hub.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: exasol__hub + description: > + Exasol-specific adapter implementation of the hub macro. Called automatically via adapter.dispatch() + when running on Exasol. See the parent hub macro for full parameter documentation. + arguments: + - name: hashkey + type: string + description: Name of the hashkey column inside the stage, used as PK of the Hub. + - name: business_keys + type: string | list of strings + description: Name(s) of the business key columns loaded into the hub. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include in the Hub. Optional, defaults to empty list. diff --git a/macros/tables/exasol/link.yml b/macros/tables/exasol/link.yml new file mode 100644 index 00000000..ac43bf98 --- /dev/null +++ b/macros/tables/exasol/link.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: exasol__link + description: > + Exasol-specific adapter implementation of the link macro. Called automatically via adapter.dispatch() + when running on Exasol. See the parent link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: Name of the link hashkey column inside the stage. + - name: foreign_hashkeys + type: list of strings + description: List of all hashkey columns inside the link that refer to other hub entities. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include in the Link. Optional, defaults to empty list. diff --git a/macros/tables/exasol/ma_sat_v0.yml b/macros/tables/exasol/ma_sat_v0.yml new file mode 100644 index 00000000..de755d38 --- /dev/null +++ b/macros/tables/exasol/ma_sat_v0.yml @@ -0,0 +1,39 @@ +version: 2 + +macros: + - name: exasol__ma_sat_v0 + description: > + Exasol-specific adapter implementation of the ma_sat_v0 macro. Called automatically via adapter.dispatch() + when running on Exasol. Uses QUALIFY for intra-batch deduplication. + See the parent ma_sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this satellite. + - name: src_ma_key + type: string | list of strings + description: Name(s) of the multi-active key column(s). + - name: src_payload + type: list of strings + description: List of descriptive attribute columns (excluding the multi-active key). + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Skips QUALIFY deduplication. Optional, default False. diff --git a/macros/tables/exasol/ma_sat_v1.yml b/macros/tables/exasol/ma_sat_v1.yml new file mode 100644 index 00000000..dccadc4b --- /dev/null +++ b/macros/tables/exasol/ma_sat_v1.yml @@ -0,0 +1,32 @@ +version: 2 + +macros: + - name: exasol__ma_sat_v1 + description: > + Exasol-specific adapter implementation of the ma_sat_v1 macro. Called automatically via adapter.dispatch() + when running on Exasol. See the parent ma_sat_v1 macro for full parameter documentation. + arguments: + - name: sat_v0 + type: string + description: Name of the underlying version 0 multi-active satellite. + - name: hashkey + type: string + description: Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: Name of the hashdiff column inside the version 0 satellite. + - name: ma_attribute + type: string | list of strings + description: Name of the multi-active attribute(s) in the v0 satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag derived from the ledts column. Optional, default False. diff --git a/macros/tables/exasol/nh_link.yml b/macros/tables/exasol/nh_link.yml new file mode 100644 index 00000000..2b58fa28 --- /dev/null +++ b/macros/tables/exasol/nh_link.yml @@ -0,0 +1,38 @@ +version: 2 + +macros: + - name: exasol__nh_link + description: > + Exasol-specific adapter implementation of the nh_link macro. Called automatically via adapter.dispatch() + when running on Exasol. See the parent nh_link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: Name of the non-historized link hashkey column inside the stage. + - name: payload + type: list of strings + description: List of descriptive attribute columns forming the payload. + - name: foreign_hashkeys + type: list of strings + description: List of all hashkey columns inside the link that refer to other hub entities. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: union_strategy + type: "'all' | 'distinct'" + description: Defines how multiple sources should be unioned. Default 'all'. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/exasol/nh_sat.yml b/macros/tables/exasol/nh_sat.yml new file mode 100644 index 00000000..a7c80d3c --- /dev/null +++ b/macros/tables/exasol/nh_sat.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: exasol__nh_sat + description: > + Exasol-specific adapter implementation of the nh_sat macro. Called automatically via adapter.dispatch() + when running on Exasol. See the parent nh_sat macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns for this non-historized satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/exasol/pit.yml b/macros/tables/exasol/pit.yml new file mode 100644 index 00000000..66215ea3 --- /dev/null +++ b/macros/tables/exasol/pit.yml @@ -0,0 +1,47 @@ +version: 2 + +macros: + - name: exasol__pit + description: > + Exasol-specific adapter implementation of the pit macro. Called automatically via adapter.dispatch() + when running on Exasol. See the parent pit macro for full parameter documentation. + arguments: + - name: pit_type + type: string + description: String to insert into the 'pit_type' column, prefixed by !. Optional. + - name: tracked_entity + type: string + description: Name of the tracked Hub entity. + - name: hashkey + type: string + description: Name of the hashkey column inside the tracked Hub. + - name: sat_names + type: list of strings + description: List of all satellite names to include in this PIT table. + - name: ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: sdts + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: custom_rsrc + type: string + description: Custom string to insert into the rsrc column. Optional. + - name: ledts + type: string + description: Name of the load-end-date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: snapshot_relation + type: string + description: Name of the snapshot relation model. + - name: snapshot_trigger_column + type: string + description: Name of the boolean column that identifies active snapshots. + - name: dimension_key + type: string + description: Desired name of the dimension key inside the PIT table. + - name: refer_to_ghost_records + type: boolean + description: Whether a NULL satellite hashkey should be replaced by the unknown key. Optional, default True. + - name: snapshot_optimization + type: boolean + description: Whether to apply snapshot optimization logic. Optional, default False. diff --git a/macros/tables/exasol/rec_track_sat.yml b/macros/tables/exasol/rec_track_sat.yml new file mode 100644 index 00000000..e60b7203 --- /dev/null +++ b/macros/tables/exasol/rec_track_sat.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: exasol__rec_track_sat + description: > + Exasol-specific adapter implementation of the rec_track_sat macro. Called automatically via adapter.dispatch() + when running on Exasol. See the parent rec_track_sat macro for full parameter documentation. + arguments: + - name: tracked_hashkey + type: string + description: The name of the hashkey column you want to track. + - name: source_models + type: dictionary + description: Dictionary with source model information, including 'rsrc_static' and optionally 'hk_column'. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: src_stg + type: string + description: Name of the source stage model. Optional, defaults to 'datavault4dbt.stg_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/exasol/ref_hub.yml b/macros/tables/exasol/ref_hub.yml new file mode 100644 index 00000000..4526f13c --- /dev/null +++ b/macros/tables/exasol/ref_hub.yml @@ -0,0 +1,23 @@ +version: 2 + +macros: + - name: exasol__ref_hub + description: > + Exasol-specific adapter implementation of the ref_hub macro. Called automatically via adapter.dispatch() + when running on Exasol. See the parent ref_hub macro for full parameter documentation. + arguments: + - name: ref_keys + type: string | list of strings + description: Name of the reference key(s) available in the source model(s). + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/exasol/ref_sat_v0.yml b/macros/tables/exasol/ref_sat_v0.yml new file mode 100644 index 00000000..17688e80 --- /dev/null +++ b/macros/tables/exasol/ref_sat_v0.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: exasol__ref_sat_v0 + description: > + Exasol-specific adapter implementation of the ref_sat_v0 macro. Called automatically via adapter.dispatch() + when running on Exasol. Uses QUALIFY for deduplication. + See the parent ref_sat_v0 macro for full parameter documentation. + arguments: + - name: parent_ref_keys + type: string | list of strings + description: Name of the reference key(s) of the parent ref_hub. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this reference satellite. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/exasol/ref_sat_v1.yml b/macros/tables/exasol/ref_sat_v1.yml new file mode 100644 index 00000000..e98bb815 --- /dev/null +++ b/macros/tables/exasol/ref_sat_v1.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: exasol__ref_sat_v1 + description: > + Exasol-specific adapter implementation of the ref_sat_v1 macro. Called automatically via adapter.dispatch() + when running on Exasol. See the parent ref_sat_v1 macro for full parameter documentation. + arguments: + - name: ref_sat_v0 + type: string + description: Name of the underlying ref_sat_v0 dbt model. + - name: ref_keys + type: string | list of strings + description: Name(s) of the reference key(s) in the underlying reference sat v0. + - name: hashdiff + type: string + description: Name of the Hashdiff column in the underlying reference sat v0. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag. Optional, default False. diff --git a/macros/tables/exasol/ref_table.yml b/macros/tables/exasol/ref_table.yml new file mode 100644 index 00000000..f44dd375 --- /dev/null +++ b/macros/tables/exasol/ref_table.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: exasol__ref_table + description: > + Exasol-specific adapter implementation of the ref_table macro. Called automatically via adapter.dispatch() + when running on Exasol. See the parent ref_table macro for full parameter documentation. + arguments: + - name: ref_hub + type: string + description: Name of the underlying ref_hub model. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ref_satellites + type: string | list of strings + description: Name(s) of the reference satellites to include. + - name: historized + type: string + description: Possible values are 'full', 'latest', or 'snapshot'. + - name: snapshot_relation + type: string + description: Only required if historized is 'snapshot'. Name of the snapshot_v1 model. + - name: snapshot_trigger_column + type: string + description: Only required if historized is 'snapshot'. Trigger column name in the snapshot relation. diff --git a/macros/tables/exasol/sat_v0.yml b/macros/tables/exasol/sat_v0.yml new file mode 100644 index 00000000..b7b43afc --- /dev/null +++ b/macros/tables/exasol/sat_v0.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: exasol__sat_v0 + description: > + Exasol-specific adapter implementation of the sat_v0 macro. Called automatically via adapter.dispatch() + when running on Exasol. Uses QUALIFY for intra-batch deduplication. + See the parent sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this satellite. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns that form the satellite payload. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Skips QUALIFY deduplication. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/exasol/sat_v1.yml b/macros/tables/exasol/sat_v1.yml new file mode 100644 index 00000000..3d68350d --- /dev/null +++ b/macros/tables/exasol/sat_v1.yml @@ -0,0 +1,32 @@ +version: 2 + +macros: + - name: exasol__sat_v1 + description: > + Exasol-specific adapter implementation of the sat_v1 macro. Called automatically via adapter.dispatch() + when running on Exasol. See the parent sat_v1 macro for full parameter documentation. + arguments: + - name: sat_v0 + type: string + description: Name of the underlying version 0 satellite. + - name: hashkey + type: string + description: Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: Name of the hashdiff column inside the version 0 satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag derived from the ledts column. Optional, default False. + - name: include_payload + type: boolean + description: Whether to include the payload columns from sat v0. Optional, default True. diff --git a/macros/tables/fabric/control_snap_v0.yml b/macros/tables/fabric/control_snap_v0.yml new file mode 100644 index 00000000..f2c2521e --- /dev/null +++ b/macros/tables/fabric/control_snap_v0.yml @@ -0,0 +1,21 @@ +version: 2 + +macros: + - name: fabric__control_snap_v0 + description: > + Microsoft Fabric-specific adapter implementation of the control_snap_v0 macro. Called automatically via + adapter.dispatch() when running on Fabric. See the parent control_snap_v0 macro for full parameter + documentation. + arguments: + - name: start_date + type: timestamp + description: Earliest timestamp for the snapshot table. Time part must be '00:00:00'. + - name: daily_snapshot_time + type: time + description: Time that daily snapshots should have (e.g. '07:00:00'). + - name: sdts_alias + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: end_date + type: timestamp + description: Latest timestamp for the snapshot table. Optional. diff --git a/macros/tables/fabric/control_snap_v1.yml b/macros/tables/fabric/control_snap_v1.yml new file mode 100644 index 00000000..dbe7e4d0 --- /dev/null +++ b/macros/tables/fabric/control_snap_v1.yml @@ -0,0 +1,20 @@ +version: 2 + +macros: + - name: fabric__control_snap_v1 + description: > + Microsoft Fabric-specific adapter implementation of the control_snap_v1 macro. Called automatically via + adapter.dispatch() when running on Fabric. See the parent control_snap_v1 macro for full parameter + documentation. + arguments: + - name: control_snap_v0 + type: string + description: The name of the underlying version 0 control snapshot table. + - name: log_logic + type: dictionary + description: > + Defining the desired durations of each granularity ('daily', 'weekly', 'monthly', 'yearly'). + Optional — if not set, all snapshots remain active. + - name: sdts_alias + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. diff --git a/macros/tables/fabric/eff_sat_v0.yml b/macros/tables/fabric/eff_sat_v0.yml new file mode 100644 index 00000000..576e047a --- /dev/null +++ b/macros/tables/fabric/eff_sat_v0.yml @@ -0,0 +1,33 @@ +version: 2 + +macros: + - name: fabric__eff_sat_v0 + description: > + Microsoft Fabric-specific adapter implementation of the eff_sat_v0 macro. Called automatically via + adapter.dispatch() when running on Fabric. Uses a lag_source_data CTE instead of QUALIFY. + See the parent eff_sat_v0 macro for full parameter documentation. + arguments: + - name: tracked_hashkey + type: string + description: Name of the hashkey column to be tracked. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: is_active_alias + type: string + description: Name of the active flag column. Optional, defaults to 'datavault4dbt.is_active_alias'. + - name: source_model + type: string + description: Name of the source model. + - name: source_is_single_batch + type: boolean + description: REQUIRED. Whether the source contains only one batch. Must be explicitly set. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be disabled. Optional. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/fabric/hub.yml b/macros/tables/fabric/hub.yml new file mode 100644 index 00000000..4be5149e --- /dev/null +++ b/macros/tables/fabric/hub.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: fabric__hub + description: > + Microsoft Fabric-specific adapter implementation of the hub macro. Called automatically via adapter.dispatch() + when running on Fabric. Uses a lag_source_data CTE approach instead of QUALIFY for deduplication. + See the parent hub macro for full parameter documentation. + arguments: + - name: hashkey + type: string + description: Name of the hashkey column inside the stage, used as PK of the Hub. + - name: business_keys + type: string | list of strings + description: Name(s) of the business key columns loaded into the hub. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include in the Hub. Optional, defaults to empty list. diff --git a/macros/tables/fabric/link.yml b/macros/tables/fabric/link.yml new file mode 100644 index 00000000..ad77281d --- /dev/null +++ b/macros/tables/fabric/link.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: fabric__link + description: > + Microsoft Fabric-specific adapter implementation of the link macro. Called automatically via adapter.dispatch() + when running on Fabric. Uses a lag_source_data CTE approach instead of QUALIFY for deduplication. + See the parent link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: Name of the link hashkey column inside the stage. + - name: foreign_hashkeys + type: list of strings + description: List of all hashkey columns inside the link that refer to other hub entities. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include in the Link. Optional, defaults to empty list. diff --git a/macros/tables/fabric/ma_sat_v0.yml b/macros/tables/fabric/ma_sat_v0.yml new file mode 100644 index 00000000..f68914e1 --- /dev/null +++ b/macros/tables/fabric/ma_sat_v0.yml @@ -0,0 +1,39 @@ +version: 2 + +macros: + - name: fabric__ma_sat_v0 + description: > + Microsoft Fabric-specific adapter implementation of the ma_sat_v0 macro. Called automatically via + adapter.dispatch() when running on Fabric. Uses a lag_source_data CTE instead of QUALIFY for + intra-batch deduplication. See the parent ma_sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this satellite. + - name: src_ma_key + type: string | list of strings + description: Name(s) of the multi-active key column(s). + - name: src_payload + type: list of strings + description: List of descriptive attribute columns (excluding the multi-active key). + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Skips lag CTE deduplication. Optional, default False. diff --git a/macros/tables/fabric/ma_sat_v1.yml b/macros/tables/fabric/ma_sat_v1.yml new file mode 100644 index 00000000..f1a44d4f --- /dev/null +++ b/macros/tables/fabric/ma_sat_v1.yml @@ -0,0 +1,32 @@ +version: 2 + +macros: + - name: fabric__ma_sat_v1 + description: > + Microsoft Fabric-specific adapter implementation of the ma_sat_v1 macro. Called automatically via + adapter.dispatch() when running on Fabric. See the parent ma_sat_v1 macro for full parameter documentation. + arguments: + - name: sat_v0 + type: string + description: Name of the underlying version 0 multi-active satellite. + - name: hashkey + type: string + description: Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: Name of the hashdiff column inside the version 0 satellite. + - name: ma_attribute + type: string | list of strings + description: Name of the multi-active attribute(s) in the v0 satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag derived from the ledts column. Optional, default False. diff --git a/macros/tables/fabric/nh_link.yml b/macros/tables/fabric/nh_link.yml new file mode 100644 index 00000000..ae77a963 --- /dev/null +++ b/macros/tables/fabric/nh_link.yml @@ -0,0 +1,38 @@ +version: 2 + +macros: + - name: fabric__nh_link + description: > + Microsoft Fabric-specific adapter implementation of the nh_link macro. Called automatically via + adapter.dispatch() when running on Fabric. See the parent nh_link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: Name of the non-historized link hashkey column inside the stage. + - name: payload + type: list of strings + description: List of descriptive attribute columns forming the payload. + - name: foreign_hashkeys + type: list of strings + description: List of all hashkey columns inside the link that refer to other hub entities. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: union_strategy + type: "'all' | 'distinct'" + description: Defines how multiple sources should be unioned. Default 'all'. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/fabric/nh_sat.yml b/macros/tables/fabric/nh_sat.yml new file mode 100644 index 00000000..7e564a9d --- /dev/null +++ b/macros/tables/fabric/nh_sat.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: fabric__nh_sat + description: > + Microsoft Fabric-specific adapter implementation of the nh_sat macro. Called automatically via + adapter.dispatch() when running on Fabric. See the parent nh_sat macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns for this non-historized satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/fabric/pit.yml b/macros/tables/fabric/pit.yml new file mode 100644 index 00000000..afa3895a --- /dev/null +++ b/macros/tables/fabric/pit.yml @@ -0,0 +1,47 @@ +version: 2 + +macros: + - name: fabric__pit + description: > + Microsoft Fabric-specific adapter implementation of the pit macro. Called automatically via adapter.dispatch() + when running on Fabric. See the parent pit macro for full parameter documentation. + arguments: + - name: pit_type + type: string + description: String to insert into the 'pit_type' column, prefixed by !. Optional. + - name: tracked_entity + type: string + description: Name of the tracked Hub entity. + - name: hashkey + type: string + description: Name of the hashkey column inside the tracked Hub. + - name: sat_names + type: list of strings + description: List of all satellite names to include in this PIT table. + - name: ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: sdts + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: custom_rsrc + type: string + description: Custom string to insert into the rsrc column. Optional. + - name: ledts + type: string + description: Name of the load-end-date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: snapshot_relation + type: string + description: Name of the snapshot relation model. + - name: snapshot_trigger_column + type: string + description: Name of the boolean column that identifies active snapshots. + - name: dimension_key + type: string + description: Desired name of the dimension key inside the PIT table. + - name: refer_to_ghost_records + type: boolean + description: Whether a NULL satellite hashkey should be replaced by the unknown key. Optional, default True. + - name: snapshot_optimization + type: boolean + description: Whether to apply snapshot optimization logic. Optional, default False. diff --git a/macros/tables/fabric/rec_track_sat.yml b/macros/tables/fabric/rec_track_sat.yml new file mode 100644 index 00000000..e7f810a3 --- /dev/null +++ b/macros/tables/fabric/rec_track_sat.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: fabric__rec_track_sat + description: > + Microsoft Fabric-specific adapter implementation of the rec_track_sat macro. Called automatically via + adapter.dispatch() when running on Fabric. See the parent rec_track_sat macro for full parameter + documentation. + arguments: + - name: tracked_hashkey + type: string + description: The name of the hashkey column you want to track. + - name: source_models + type: dictionary + description: Dictionary with source model information, including 'rsrc_static' and optionally 'hk_column'. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: src_stg + type: string + description: Name of the source stage model. Optional, defaults to 'datavault4dbt.stg_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/fabric/ref_hub.yml b/macros/tables/fabric/ref_hub.yml new file mode 100644 index 00000000..11979e8c --- /dev/null +++ b/macros/tables/fabric/ref_hub.yml @@ -0,0 +1,23 @@ +version: 2 + +macros: + - name: fabric__ref_hub + description: > + Microsoft Fabric-specific adapter implementation of the ref_hub macro. Called automatically via + adapter.dispatch() when running on Fabric. See the parent ref_hub macro for full parameter documentation. + arguments: + - name: ref_keys + type: string | list of strings + description: Name of the reference key(s) available in the source model(s). + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/fabric/ref_sat_v0.yml b/macros/tables/fabric/ref_sat_v0.yml new file mode 100644 index 00000000..de004de2 --- /dev/null +++ b/macros/tables/fabric/ref_sat_v0.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: fabric__ref_sat_v0 + description: > + Microsoft Fabric-specific adapter implementation of the ref_sat_v0 macro. Called automatically via + adapter.dispatch() when running on Fabric. Uses a lag_source_data CTE instead of QUALIFY for deduplication. + See the parent ref_sat_v0 macro for full parameter documentation. + arguments: + - name: parent_ref_keys + type: string | list of strings + description: Name of the reference key(s) of the parent ref_hub. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this reference satellite. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/fabric/ref_sat_v1.yml b/macros/tables/fabric/ref_sat_v1.yml new file mode 100644 index 00000000..e346617f --- /dev/null +++ b/macros/tables/fabric/ref_sat_v1.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: fabric__ref_sat_v1 + description: > + Microsoft Fabric-specific adapter implementation of the ref_sat_v1 macro. Called automatically via + adapter.dispatch() when running on Fabric. See the parent ref_sat_v1 macro for full parameter documentation. + arguments: + - name: ref_sat_v0 + type: string + description: Name of the underlying ref_sat_v0 dbt model. + - name: ref_keys + type: string | list of strings + description: Name(s) of the reference key(s) in the underlying reference sat v0. + - name: hashdiff + type: string + description: Name of the Hashdiff column in the underlying reference sat v0. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag. Optional, default False. diff --git a/macros/tables/fabric/ref_table.yml b/macros/tables/fabric/ref_table.yml new file mode 100644 index 00000000..20e41969 --- /dev/null +++ b/macros/tables/fabric/ref_table.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: fabric__ref_table + description: > + Microsoft Fabric-specific adapter implementation of the ref_table macro. Called automatically via + adapter.dispatch() when running on Fabric. See the parent ref_table macro for full parameter documentation. + arguments: + - name: ref_hub + type: string + description: Name of the underlying ref_hub model. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ref_satellites + type: string | list of strings + description: Name(s) of the reference satellites to include. + - name: historized + type: string + description: Possible values are 'full', 'latest', or 'snapshot'. + - name: snapshot_relation + type: string + description: Only required if historized is 'snapshot'. Name of the snapshot_v1 model. + - name: snapshot_trigger_column + type: string + description: Only required if historized is 'snapshot'. Trigger column name in the snapshot relation. diff --git a/macros/tables/fabric/sat_v0.yml b/macros/tables/fabric/sat_v0.yml new file mode 100644 index 00000000..963be5ae --- /dev/null +++ b/macros/tables/fabric/sat_v0.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: fabric__sat_v0 + description: > + Microsoft Fabric-specific adapter implementation of the sat_v0 macro. Called automatically via + adapter.dispatch() when running on Fabric. Uses a lag_source_data CTE instead of QUALIFY for + intra-batch deduplication. See the parent sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this satellite. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns that form the satellite payload. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Skips lag CTE deduplication. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/fabric/sat_v1.yml b/macros/tables/fabric/sat_v1.yml new file mode 100644 index 00000000..238536c1 --- /dev/null +++ b/macros/tables/fabric/sat_v1.yml @@ -0,0 +1,32 @@ +version: 2 + +macros: + - name: fabric__sat_v1 + description: > + Microsoft Fabric-specific adapter implementation of the sat_v1 macro. Called automatically via + adapter.dispatch() when running on Fabric. See the parent sat_v1 macro for full parameter documentation. + arguments: + - name: sat_v0 + type: string + description: Name of the underlying version 0 satellite. + - name: hashkey + type: string + description: Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: Name of the hashdiff column inside the version 0 satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag derived from the ledts column. Optional, default False. + - name: include_payload + type: boolean + description: Whether to include the payload columns from sat v0. Optional, default True. diff --git a/macros/tables/hub.md b/macros/tables/hub.md new file mode 100644 index 00000000..5ef8e896 --- /dev/null +++ b/macros/tables/hub.md @@ -0,0 +1,41 @@ +{% docs hub %} + +## Hub + +Creates a Hub entity in the Raw Data Vault. A Hub captures and stores the unique business keys for a business concept, +along with the first load date and record source. Each business key combination is stored only once. + +Features: +- Loadable by multiple sources +- Supports multiple updates per batch and therefore initial loading +- Can use a dynamic high-water-mark to optimize loading performance of multiple loads +- Allows source mappings for deviations between source column names and hub column names + +### Usage + +```jinja +{{ datavault4dbt.hub( + hashkey='hk_account_h', + business_keys='account_id', + source_models={ + 'stg_account_crm': { + 'bk_columns': 'account_id', + 'rsrc_static': '*/CRM/Accounts/*' + }, + 'stg_account_erp': { + 'bk_columns': 'account_id', + 'rsrc_static': '*/ERP/Accounts/*' + } + } +) }} +``` + +### Metadata block usage + +```jinja +{{ datavault4dbt.hub(yaml_metadata=meta) }} +``` + +Where `meta` is a YAML-parsed dictionary containing the same keys as the macro parameters. + +{% enddocs %} diff --git a/macros/tables/hub.yml b/macros/tables/hub.yml new file mode 100644 index 00000000..bc13190a --- /dev/null +++ b/macros/tables/hub.yml @@ -0,0 +1,53 @@ +version: 2 + +macros: + - name: hub + description: '{{ doc("hub") }}' + arguments: + - name: yaml_metadata + type: dictionary + description: > + Optional. A YAML metadata block that passes all parameters as a single dictionary instead + of specifying each one directly in the macro call. + - name: hashkey + type: string + description: > + Name of the hashkey column inside the stage, that should be used as PK of the Hub. + Examples: 'hk_account_h' — This hashkey column was created before inside the corresponding staging area, using the stage macro. + - name: business_keys + type: string | list of strings + description: > + Name(s) of the business key columns that should be loaded into the hub and are the input of the hashkey column. + Needs to be available inside the stage model. If the names differ between multiple sources, define here how the business + keys should be called inside the final hub model. The actual input column names need to be defined inside the + 'source_model' parameter then. + Examples: 'account_key' (single), ['account_key', 'account_number'] (composite). + - name: source_models + type: dictionary + description: > + Dictionary with information about the source models. The keys of the dict are the names of the source models, + and the value of each source model is another dictionary. This inner dictionary requires the key 'bk_columns' + (which contains the name of the business keys of that source model), and can have the optional keys 'hk_column' + and 'rsrc_static'. The 'rsrc_static' attribute defines a STRING or list of strings containing all the patterns + of the record_source field that remain the same over all loads of one source. Wildcards (e.g. '*') can be used + to match variable parts. + - name: src_ldts + type: string + description: > + Name of the ldts column inside the source models. Is optional, will use the global variable 'datavault4dbt.ldts_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: src_rsrc + type: string + description: > + Name of the rsrc column inside the source models. Is optional, will use the global variable 'datavault4dbt.rsrc_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: disable_hwm + type: boolean + description: > + Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: > + Additional columns from source system or derived columns which should be part of Hub. Useful when you have to + deviate from the default Hub structure. The columns need to be available in all source models which are used + for the Hub. Optional parameter, defaults to empty list. diff --git a/macros/tables/link.md b/macros/tables/link.md new file mode 100644 index 00000000..51466f39 --- /dev/null +++ b/macros/tables/link.md @@ -0,0 +1,29 @@ +{% docs link %} + +## Link + +Creates a Link entity in the Raw Data Vault. A Link connects two or more entities, or an entity with itself. +It can be loaded by one or more source staging tables if multiple sources share the same business definitions. +If multiple sources are used, they must all have the same number of foreign keys. + +### Usage + +```jinja +{{ datavault4dbt.link( + link_hashkey='hk_account_contact_l', + foreign_hashkeys=['hk_account_h', 'hk_contact_h'], + source_models={ + 'stg_account_contact': { + 'rsrc_static': '*/CRM/AccountContact/*' + } + } +) }} +``` + +### Metadata block usage + +```jinja +{{ datavault4dbt.link(yaml_metadata=meta) }} +``` + +{% enddocs %} diff --git a/macros/tables/link.yml b/macros/tables/link.yml new file mode 100644 index 00000000..1c73741a --- /dev/null +++ b/macros/tables/link.yml @@ -0,0 +1,52 @@ +version: 2 + +macros: + - name: link + description: '{{ doc("link") }}' + arguments: + - name: yaml_metadata + type: dictionary + description: > + Optional. A YAML metadata block that passes all parameters as a single dictionary instead + of specifying each one directly in the macro call. + - name: link_hashkey + type: string + description: > + Name of the link hashkey column inside the stage. Should get calculated out of all business keys inside the link. + Examples: 'hk_account_contact_l' — This hashkey column belongs to the link between account and contact, and + was created at the staging layer by the stage macro. + - name: foreign_hashkeys + type: list of strings + description: > + List of all hashkey columns inside the link, that refer to other hub entities. All hashkey columns must + be available inside the stage area. + Examples: ['hk_account_h', 'hk_contact_h'] — The link between account and contact needs to contain both + the hashkey of account and contact to enable joins to the corresponding hub entities. + - name: source_models + type: dictionary + description: > + Dictionary with information about the source models. The keys of the dict are the names of the source models, + and the value of each source model is another dictionary. This inner dictionary requires the key 'rsrc_static', + and optionally the keys 'hk_column' and 'fk_columns'. The 'rsrc_static' attribute defines a STRING or list of + strings containing all the patterns of the record_source field that remain the same over all loads of one source. + Wildcards (e.g. '*') can be used to match variable parts. + - name: src_ldts + type: string + description: > + Name of the ldts column inside the source models. Is optional, will use the global variable 'datavault4dbt.ldts_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: src_rsrc + type: string + description: > + Name of the rsrc column inside the source models. Is optional, will use the global variable 'datavault4dbt.rsrc_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: disable_hwm + type: boolean + description: > + Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: > + Additional columns from source system or derived columns which should be part of Link. Useful when you have to + deviate from the default Link structure. The columns need to be available in all source models which are used + for the Link. Optional parameter, defaults to empty list. diff --git a/macros/tables/ma_sat_v0.md b/macros/tables/ma_sat_v0.md new file mode 100644 index 00000000..14d29872 --- /dev/null +++ b/macros/tables/ma_sat_v0.md @@ -0,0 +1,47 @@ +{% docs ma_sat_v0 %} + +## Multi-Active Satellite v0 + +Creates a multi-active satellite version 0, materialized as an incremental table. Applied on top of the staging layer, +connected to either a Hub or a Link. A multi-active satellite stores multiple concurrently valid records per parent +hashkey, distinguished by a multi-active key (e.g. multiple phone numbers per contact). + +On top of each version 0 multi-active satellite, a version 1 should be created using the ma_sat_v1 macro. +If a stage model is defined as multi-active, all satellites out of that stage model must be implemented as +multi-active satellites. + +Features: +- Can handle multiple updates per batch, without losing intermediate changes — initial loading is supported. +- Uses a dynamic high-water-mark to optimize loading performance of multiple loads. + +### Usage + +```jinja +{{ datavault4dbt.ma_sat_v0( + parent_hashkey='hk_contact_h', + src_hashdiff='hd_contact_phonenumber_s', + src_ma_key='phonetype', + src_payload=['phone_number', 'is_primary'], + source_model='stage_contact' +) }} +``` + +### With composite multi-active key + +```jinja +{{ datavault4dbt.ma_sat_v0( + parent_hashkey='hk_contact_h', + src_hashdiff='hd_contact_phonenumber_s', + src_ma_key=['phonetype', 'iid'], + src_payload=['phone_number'], + source_model='stage_contact' +) }} +``` + +### Metadata block usage + +```jinja +{{ datavault4dbt.ma_sat_v0(yaml_metadata=meta) }} +``` + +{% enddocs %} diff --git a/macros/tables/ma_sat_v0.yml b/macros/tables/ma_sat_v0.yml new file mode 100644 index 00000000..b6de1786 --- /dev/null +++ b/macros/tables/ma_sat_v0.yml @@ -0,0 +1,64 @@ +version: 2 + +macros: + - name: ma_sat_v0 + description: '{{ doc("ma_sat_v0") }}' + arguments: + - name: yaml_metadata + type: dictionary + description: > + Optional. A YAML metadata block that passes all parameters as a single dictionary instead + of specifying each one directly in the macro call. + - name: parent_hashkey + type: string + description: > + Name of the hashkey column inside the stage of the object that this satellite is attached to. + Examples: 'hk_account_h' (attached to hub account), 'hk_account_contact_l' (attached to a link). + - name: src_hashdiff + type: string + description: > + Name of the hashdiff column of this satellite, that was created inside the staging area and is + calculated out of the entire payload of this satellite. The stage must hold one hashdiff per + satellite entity. Examples: 'hd_account_data_sfdc_s'. + - name: src_ma_key + type: string | list of strings + description: > + Name(s) of the multi-active keys inside the staging area. Need to be the same ones as defined in the stage model. + Examples: 'phonetype' (single), ['phonetype', 'company'] (composite — the combination of both is treated as the key). + - name: src_payload + type: list of strings + description: > + A list of all the descriptive attributes that should be included in this satellite. Needs to be the columns that + are fed into the hashdiff calculation of this satellite. Do not include the multi-active key in the payload of a + multi-active satellite, it is included automatically! + Examples: ['name', 'address', 'country', 'phone', 'email']. + - name: source_model + type: string + description: > + Name of the underlying staging model, must be available inside dbt as a model. + Examples: 'stage_account' — This satellite is loaded out of the stage for account. + - name: src_ldts + type: string + description: > + Name of the ldts column inside the source model. Is optional, will use the global variable 'datavault4dbt.ldts_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: src_rsrc + type: string + description: > + Name of the rsrc column inside the source model. Is optional, will use the global variable 'datavault4dbt.rsrc_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: additional_columns + type: list | string + description: > + Additional columns from source system or derived columns which should be part of the ma_sat. + The columns need to be available in all source models. Optional parameter, defaults to empty list. + - name: disable_hwm + type: boolean + description: > + Whether the automatic application of a High-Water Mark (HWM) should be disabled or not. + Optional parameter, defaults to False. + - name: source_is_single_batch + type: boolean + description: > + Boosts performance by disabling QUALIFY statement. Only activate this if you made sure that the underlying + staging model only holds one row per entry. Optional parameter, defaults to False. diff --git a/macros/tables/ma_sat_v1.md b/macros/tables/ma_sat_v1.md new file mode 100644 index 00000000..a6824797 --- /dev/null +++ b/macros/tables/ma_sat_v1.md @@ -0,0 +1,43 @@ +{% docs ma_sat_v1 %} + +## Multi-Active Satellite v1 + +Calculates the load end dates for multi-active data, based on a multi-active attribute. Must be based on a version 0 +multi-active satellite, that would then hold multiple records per hashkey+ldts combination. + +A version 1 multi-active satellite should be materialized as a view by default. + +Features: +- Calculates virtualized load-end-dates to correctly identify multiple active records per batch. +- Enforces insert-only approach by view materialization. +- Allows multiple attributes to be used as the multi-active-attribute. + +### Usage + +```jinja +{{ datavault4dbt.ma_sat_v1( + sat_v0='contact_phonenumber_0_s', + hashkey='hk_contact_h', + hashdiff='hd_contact_phonenumber_s', + ma_attribute='phone_type' +) }} +``` + +### With composite multi-active attribute + +```jinja +{{ datavault4dbt.ma_sat_v1( + sat_v0='contact_phonenumber_0_s', + hashkey='hk_contact_h', + hashdiff='hd_contact_phonenumber_s', + ma_attribute=['phone_type', 'iid'] +) }} +``` + +### Metadata block usage + +```jinja +{{ datavault4dbt.ma_sat_v1(yaml_metadata=meta) }} +``` + +{% enddocs %} diff --git a/macros/tables/ma_sat_v1.yml b/macros/tables/ma_sat_v1.yml new file mode 100644 index 00000000..5d9a38ee --- /dev/null +++ b/macros/tables/ma_sat_v1.yml @@ -0,0 +1,56 @@ +version: 2 + +macros: + - name: ma_sat_v1 + description: '{{ doc("ma_sat_v1") }}' + arguments: + - name: yaml_metadata + type: dictionary + description: > + Optional. A YAML metadata block that passes all parameters as a single dictionary instead + of specifying each one directly in the macro call. + - name: sat_v0 + type: string + description: > + Name of the underlying version 0 multi-active satellite. + Examples: 'contact_phonenumber_0_s' — This satellite would be the version 1 satellite of the underlying + version 0 phone number satellite for contacts. + - name: hashkey + type: string + description: > + Name of the parent hashkey column inside the version 0 satellite. Would either be the hashkey of a + hub or a link. Needs to be similar to the 'parent_hashkey' parameter inside the sat_v0 model. + Examples: 'hk_contact_h' (hub), 'hk_order_contact_l' (link). + - name: hashdiff + type: string + description: > + Name of the hashdiff column inside the underlying version 0 satellite. Needs to be similar to the + 'src_hashdiff' parameter inside the sat_v0 model. Must not include the ma_attribute in calculation. + Examples: 'hd_contact_phonenumber_s'. + - name: ma_attribute + type: string | list of strings + description: > + Name of the multi active attribute inside the v0 satellite. This needs to be identified under the + requirement that the combination of hashkey + ldts + ma_attribute is unique over the entire stage/satellite. + Examples: 'phone_type' (single), ['phone_type', 'iid'] (composite — needed when multiple records share the same phone_type). + - name: src_ldts + type: string + description: > + Name of the ldts column inside the source models. Is optional, will use the global variable 'datavault4dbt.ldts_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: src_rsrc + type: string + description: > + Name of the rsrc column inside the source models. Is optional, will use the global variable 'datavault4dbt.rsrc_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: ledts_alias + type: string + description: > + Desired alias for the load end date column. Is optional, will use the global variable 'datavault4dbt.ledts_alias' + if set here. + - name: add_is_current_flag + type: boolean + description: > + Optional parameter to add a new column to the v1 sat based on the load end date timestamp (ledts). Default is false. + If set to true it will add this is_current flag to the v1 sat. For each record this column will be set to true if + the load end date timestamp is equal to the variable end of all times. If not, the record is not current. diff --git a/macros/tables/nh_link.md b/macros/tables/nh_link.md new file mode 100644 index 00000000..212611a1 --- /dev/null +++ b/macros/tables/nh_link.md @@ -0,0 +1,33 @@ +{% docs nh_link %} + +## Non-Historized Link + +Creates a non-historized (formerly transactional) link entity, connecting two or more entities, or a transactional +fact of one entity. It can be loaded by one or more source staging tables if multiple sources share the same +business definitions. + +In the background a non-historized link uses exactly the same loading logic as a regular link, but adds the +descriptive attributes as additional payload. + +### Usage + +```jinja +{{ datavault4dbt.nh_link( + link_hashkey='hk_transaction_account_nl', + foreign_hashkeys=['hk_transaction_h', 'hk_account_h'], + payload=['currency_isocode', 'amount', 'purpose', 'transaction_date'], + source_models={ + 'stg_transaction': { + 'rsrc_static': '*/ERP/Transactions/*' + } + } +) }} +``` + +### Metadata block usage + +```jinja +{{ datavault4dbt.nh_link(yaml_metadata=meta) }} +``` + +{% enddocs %} diff --git a/macros/tables/nh_link.yml b/macros/tables/nh_link.yml new file mode 100644 index 00000000..d57c789c --- /dev/null +++ b/macros/tables/nh_link.yml @@ -0,0 +1,70 @@ +version: 2 + +macros: + - name: nh_link + description: '{{ doc("nh_link") }}' + arguments: + - name: yaml_metadata + type: dictionary + description: > + Optional. A YAML metadata block that passes all parameters as a single dictionary instead + of specifying each one directly in the macro call. + - name: link_hashkey + type: string + description: > + Name of the non-historized link hashkey column inside the stage. Should get calculated out of all + business keys inside the link. + Examples: 'hk_transaction_account_nl' — This hashkey column belongs to the non-historized link + between transaction and account, and was created at the staging layer by the stage macro. + - name: payload + type: list of strings + description: > + A list of all the descriptive attributes that should be the payload of this non-historized link. + If the names differ between source models, this list defines how the columns are named inside the + result non-historized link. The mapping which columns to use from which source model must then be + defined inside the 'payload' key in 'source_models'. + Examples: ['currency_isocode', 'amount', 'purpose', 'transaction_date']. + - name: source_models + type: dictionary + description: > + Dictionary with information about the source models. The keys of the dict are the names of the source + models, and the value of each source model is another dictionary. This inner dictionary optionally has + the keys 'hk_column', 'fk_columns', 'payload' and 'rsrc_static'. The 'rsrc_static' attribute defines + a STRING or list of strings containing all the patterns of the record_source field that remain the same + over all loads of one source. Wildcards (e.g. '*') can be used to match variable parts. + - name: foreign_hashkeys + type: list of strings + description: > + List of all hashkey columns inside the non-historized link, that refer to other hub entities. + All hashkey columns must be available inside the stage area. + Examples: ['hk_transaction_h', 'hk_account_h']. + - name: src_ldts + type: string + description: > + Name of the ldts column inside the source models. Is optional, will use the global variable 'datavault4dbt.ldts_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: src_rsrc + type: string + description: > + Name of the rsrc column inside the source models. Is optional, will use the global variable 'datavault4dbt.rsrc_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: disable_hwm + type: boolean + description: > + Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: > + Whether the source contains only one batch. Optional, default False. + - name: union_strategy + type: "'all' | 'distinct'" + description: > + Defines how multiple sources should be unioned. 'all' will result in a UNION ALL and represents the + default value. Should only be changed if you have duplicates across source systems and do not want + to deduplicate them upfront. + - name: additional_columns + type: list | string + description: > + Additional columns from source system or derived columns which should be part of NH-Link. Useful when + you have to deviate from the default NH-Link structure. The columns need to be available in all source + models which are used for the NH-Link. Optional parameter, defaults to empty list. diff --git a/macros/tables/nh_sat.md b/macros/tables/nh_sat.md new file mode 100644 index 00000000..c0a5e981 --- /dev/null +++ b/macros/tables/nh_sat.md @@ -0,0 +1,28 @@ +{% docs nh_sat %} + +## Non-Historized Satellite + +Creates a non-historized satellite, materialized as an incremental table. Applied on top of the staging layer, +connected to either a Hub or a Link. Besides the missing hashdiff, a non-historized satellite applies the same +loading logic as a regular version 0 satellite. Each satellite can only be loaded by one source model. + +Features: +- High-performance loading of non-historized satellite data. + +### Usage + +```jinja +{{ datavault4dbt.nh_sat( + parent_hashkey='hk_account_h', + src_payload=['name', 'address', 'country', 'phone', 'email'], + source_model='stage_account' +) }} +``` + +### Metadata block usage + +```jinja +{{ datavault4dbt.nh_sat(yaml_metadata=meta) }} +``` + +{% enddocs %} diff --git a/macros/tables/nh_sat.yml b/macros/tables/nh_sat.yml new file mode 100644 index 00000000..33d58099 --- /dev/null +++ b/macros/tables/nh_sat.yml @@ -0,0 +1,45 @@ +version: 2 + +macros: + - name: nh_sat + description: '{{ doc("nh_sat") }}' + arguments: + - name: yaml_metadata + type: dictionary + description: > + Optional. A YAML metadata block that passes all parameters as a single dictionary instead + of specifying each one directly in the macro call. + - name: parent_hashkey + type: string + description: > + Name of the hashkey column inside the stage of the object that this satellite is attached to. + Examples: 'hk_account_h' (attached to hub account), 'hk_account_contact_l' (attached to a link). + - name: src_payload + type: list of strings + description: > + A list of all the descriptive attributes that should be included in this satellite. + Examples: ['name', 'address', 'country', 'phone', 'email']. + - name: source_model + type: string + description: > + Name of the underlying staging model, must be available inside dbt as a model. + Examples: 'stage_account' — This satellite is loaded out of the stage for account. + - name: src_ldts + type: string + description: > + Name of the ldts column inside the source model. Is optional, will use the global variable 'datavault4dbt.ldts_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: src_rsrc + type: string + description: > + Name of the rsrc column inside the source model. Is optional, will use the global variable 'datavault4dbt.rsrc_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: source_is_single_batch + type: boolean + description: > + Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: > + Additional columns from source system or derived columns which should be part of the nh_sat. + The columns need to be available in all source models. Optional parameter, defaults to empty list. diff --git a/macros/tables/oracle/control_snap_v0.yml b/macros/tables/oracle/control_snap_v0.yml new file mode 100644 index 00000000..d5890e5e --- /dev/null +++ b/macros/tables/oracle/control_snap_v0.yml @@ -0,0 +1,21 @@ +version: 2 + +macros: + - name: oracle__control_snap_v0 + description: > + Oracle-specific adapter implementation of the control_snap_v0 macro. Called automatically via + adapter.dispatch() when running on Oracle. See the parent control_snap_v0 macro for full parameter + documentation. + arguments: + - name: start_date + type: timestamp + description: Earliest timestamp for the snapshot table. Time part must be '00:00:00'. + - name: daily_snapshot_time + type: time + description: Time that daily snapshots should have (e.g. '07:00:00'). + - name: sdts_alias + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: end_date + type: timestamp + description: Latest timestamp for the snapshot table. Optional. diff --git a/macros/tables/oracle/control_snap_v1.yml b/macros/tables/oracle/control_snap_v1.yml new file mode 100644 index 00000000..9c51f9cb --- /dev/null +++ b/macros/tables/oracle/control_snap_v1.yml @@ -0,0 +1,20 @@ +version: 2 + +macros: + - name: oracle__control_snap_v1 + description: > + Oracle-specific adapter implementation of the control_snap_v1 macro. Called automatically via + adapter.dispatch() when running on Oracle. See the parent control_snap_v1 macro for full parameter + documentation. + arguments: + - name: control_snap_v0 + type: string + description: The name of the underlying version 0 control snapshot table. + - name: log_logic + type: dictionary + description: > + Defining the desired durations of each granularity ('daily', 'weekly', 'monthly', 'yearly'). + Optional — if not set, all snapshots remain active. + - name: sdts_alias + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. diff --git a/macros/tables/oracle/eff_sat_v0.yml b/macros/tables/oracle/eff_sat_v0.yml new file mode 100644 index 00000000..1ca1475d --- /dev/null +++ b/macros/tables/oracle/eff_sat_v0.yml @@ -0,0 +1,33 @@ +version: 2 + +macros: + - name: oracle__eff_sat_v0 + description: > + Oracle-specific adapter implementation of the eff_sat_v0 macro. Called automatically via adapter.dispatch() + when running on Oracle. Uses a lag_source_data CTE instead of QUALIFY. + See the parent eff_sat_v0 macro for full parameter documentation. + arguments: + - name: tracked_hashkey + type: string + description: Name of the hashkey column to be tracked. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: is_active_alias + type: string + description: Name of the active flag column. Optional, defaults to 'datavault4dbt.is_active_alias'. + - name: source_model + type: string + description: Name of the source model. + - name: source_is_single_batch + type: boolean + description: REQUIRED. Whether the source contains only one batch. Must be explicitly set. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be disabled. Optional. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/oracle/hub.yml b/macros/tables/oracle/hub.yml new file mode 100644 index 00000000..a5035488 --- /dev/null +++ b/macros/tables/oracle/hub.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: oracle__hub + description: > + Oracle-specific adapter implementation of the hub macro. Called automatically via adapter.dispatch() + when running on Oracle. Uses a lag_source_data CTE approach instead of QUALIFY for deduplication. + See the parent hub macro for full parameter documentation. + arguments: + - name: hashkey + type: string + description: Name of the hashkey column inside the stage, used as PK of the Hub. + - name: business_keys + type: string | list of strings + description: Name(s) of the business key columns loaded into the hub. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include in the Hub. Optional, defaults to empty list. diff --git a/macros/tables/oracle/link.yml b/macros/tables/oracle/link.yml new file mode 100644 index 00000000..a2f855a8 --- /dev/null +++ b/macros/tables/oracle/link.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: oracle__link + description: > + Oracle-specific adapter implementation of the link macro. Called automatically via adapter.dispatch() + when running on Oracle. Uses a lag_source_data CTE approach instead of QUALIFY for deduplication. + See the parent link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: Name of the link hashkey column inside the stage. + - name: foreign_hashkeys + type: list of strings + description: List of all hashkey columns inside the link that refer to other hub entities. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include in the Link. Optional, defaults to empty list. diff --git a/macros/tables/oracle/ma_sat_v0.yml b/macros/tables/oracle/ma_sat_v0.yml new file mode 100644 index 00000000..bb3a669e --- /dev/null +++ b/macros/tables/oracle/ma_sat_v0.yml @@ -0,0 +1,39 @@ +version: 2 + +macros: + - name: oracle__ma_sat_v0 + description: > + Oracle-specific adapter implementation of the ma_sat_v0 macro. Called automatically via adapter.dispatch() + when running on Oracle. Uses a lag_source_data CTE instead of QUALIFY for intra-batch deduplication. + See the parent ma_sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this satellite. + - name: src_ma_key + type: string | list of strings + description: Name(s) of the multi-active key column(s). + - name: src_payload + type: list of strings + description: List of descriptive attribute columns (excluding the multi-active key). + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Skips lag CTE deduplication. Optional, default False. diff --git a/macros/tables/oracle/ma_sat_v1.yml b/macros/tables/oracle/ma_sat_v1.yml new file mode 100644 index 00000000..1f2e49ac --- /dev/null +++ b/macros/tables/oracle/ma_sat_v1.yml @@ -0,0 +1,32 @@ +version: 2 + +macros: + - name: oracle__ma_sat_v1 + description: > + Oracle-specific adapter implementation of the ma_sat_v1 macro. Called automatically via adapter.dispatch() + when running on Oracle. See the parent ma_sat_v1 macro for full parameter documentation. + arguments: + - name: sat_v0 + type: string + description: Name of the underlying version 0 multi-active satellite. + - name: hashkey + type: string + description: Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: Name of the hashdiff column inside the version 0 satellite. + - name: ma_attribute + type: string | list of strings + description: Name of the multi-active attribute(s) in the v0 satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag derived from the ledts column. Optional, default False. diff --git a/macros/tables/oracle/nh_link.yml b/macros/tables/oracle/nh_link.yml new file mode 100644 index 00000000..d01f9cad --- /dev/null +++ b/macros/tables/oracle/nh_link.yml @@ -0,0 +1,38 @@ +version: 2 + +macros: + - name: oracle__nh_link + description: > + Oracle-specific adapter implementation of the nh_link macro. Called automatically via adapter.dispatch() + when running on Oracle. See the parent nh_link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: Name of the non-historized link hashkey column inside the stage. + - name: payload + type: list of strings + description: List of descriptive attribute columns forming the payload. + - name: foreign_hashkeys + type: list of strings + description: List of all hashkey columns inside the link that refer to other hub entities. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: union_strategy + type: "'all' | 'distinct'" + description: Defines how multiple sources should be unioned. Default 'all'. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/oracle/nh_sat.yml b/macros/tables/oracle/nh_sat.yml new file mode 100644 index 00000000..ec853f8a --- /dev/null +++ b/macros/tables/oracle/nh_sat.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: oracle__nh_sat + description: > + Oracle-specific adapter implementation of the nh_sat macro. Called automatically via adapter.dispatch() + when running on Oracle. See the parent nh_sat macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns for this non-historized satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/oracle/pit.yml b/macros/tables/oracle/pit.yml new file mode 100644 index 00000000..fcdff874 --- /dev/null +++ b/macros/tables/oracle/pit.yml @@ -0,0 +1,47 @@ +version: 2 + +macros: + - name: oracle__pit + description: > + Oracle-specific adapter implementation of the pit macro. Called automatically via adapter.dispatch() + when running on Oracle. See the parent pit macro for full parameter documentation. + arguments: + - name: pit_type + type: string + description: String to insert into the 'pit_type' column, prefixed by !. Optional. + - name: tracked_entity + type: string + description: Name of the tracked Hub entity. + - name: hashkey + type: string + description: Name of the hashkey column inside the tracked Hub. + - name: sat_names + type: list of strings + description: List of all satellite names to include in this PIT table. + - name: ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: sdts + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: custom_rsrc + type: string + description: Custom string to insert into the rsrc column. Optional. + - name: ledts + type: string + description: Name of the load-end-date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: snapshot_relation + type: string + description: Name of the snapshot relation model. + - name: snapshot_trigger_column + type: string + description: Name of the boolean column that identifies active snapshots. + - name: dimension_key + type: string + description: Desired name of the dimension key inside the PIT table. + - name: refer_to_ghost_records + type: boolean + description: Whether a NULL satellite hashkey should be replaced by the unknown key. Optional, default True. + - name: snapshot_optimization + type: boolean + description: Whether to apply snapshot optimization logic. Optional, default False. diff --git a/macros/tables/oracle/rec_track_sat.yml b/macros/tables/oracle/rec_track_sat.yml new file mode 100644 index 00000000..8912b9ee --- /dev/null +++ b/macros/tables/oracle/rec_track_sat.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: oracle__rec_track_sat + description: > + Oracle-specific adapter implementation of the rec_track_sat macro. Called automatically via adapter.dispatch() + when running on Oracle. See the parent rec_track_sat macro for full parameter documentation. + arguments: + - name: tracked_hashkey + type: string + description: The name of the hashkey column you want to track. + - name: source_models + type: dictionary + description: Dictionary with source model information, including 'rsrc_static' and optionally 'hk_column'. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: src_stg + type: string + description: Name of the source stage model. Optional, defaults to 'datavault4dbt.stg_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/oracle/ref_hub.yml b/macros/tables/oracle/ref_hub.yml new file mode 100644 index 00000000..45a4be55 --- /dev/null +++ b/macros/tables/oracle/ref_hub.yml @@ -0,0 +1,23 @@ +version: 2 + +macros: + - name: oracle__ref_hub + description: > + Oracle-specific adapter implementation of the ref_hub macro. Called automatically via adapter.dispatch() + when running on Oracle. See the parent ref_hub macro for full parameter documentation. + arguments: + - name: ref_keys + type: string | list of strings + description: Name of the reference key(s) available in the source model(s). + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/oracle/ref_sat_v0.yml b/macros/tables/oracle/ref_sat_v0.yml new file mode 100644 index 00000000..014c3a3a --- /dev/null +++ b/macros/tables/oracle/ref_sat_v0.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: oracle__ref_sat_v0 + description: > + Oracle-specific adapter implementation of the ref_sat_v0 macro. Called automatically via adapter.dispatch() + when running on Oracle. Uses a lag_source_data CTE instead of QUALIFY for deduplication. + See the parent ref_sat_v0 macro for full parameter documentation. + arguments: + - name: parent_ref_keys + type: string | list of strings + description: Name of the reference key(s) of the parent ref_hub. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this reference satellite. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/oracle/ref_sat_v1.yml b/macros/tables/oracle/ref_sat_v1.yml new file mode 100644 index 00000000..eed5eb4d --- /dev/null +++ b/macros/tables/oracle/ref_sat_v1.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: oracle__ref_sat_v1 + description: > + Oracle-specific adapter implementation of the ref_sat_v1 macro. Called automatically via adapter.dispatch() + when running on Oracle. See the parent ref_sat_v1 macro for full parameter documentation. + arguments: + - name: ref_sat_v0 + type: string + description: Name of the underlying ref_sat_v0 dbt model. + - name: ref_keys + type: string | list of strings + description: Name(s) of the reference key(s) in the underlying reference sat v0. + - name: hashdiff + type: string + description: Name of the Hashdiff column in the underlying reference sat v0. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag. Optional, default False. diff --git a/macros/tables/oracle/ref_table.yml b/macros/tables/oracle/ref_table.yml new file mode 100644 index 00000000..649b9002 --- /dev/null +++ b/macros/tables/oracle/ref_table.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: oracle__ref_table + description: > + Oracle-specific adapter implementation of the ref_table macro. Called automatically via adapter.dispatch() + when running on Oracle. See the parent ref_table macro for full parameter documentation. + arguments: + - name: ref_hub + type: string + description: Name of the underlying ref_hub model. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ref_satellites + type: string | list of strings + description: Name(s) of the reference satellites to include. + - name: historized + type: string + description: Possible values are 'full', 'latest', or 'snapshot'. + - name: snapshot_relation + type: string + description: Only required if historized is 'snapshot'. Name of the snapshot_v1 model. + - name: snapshot_trigger_column + type: string + description: Only required if historized is 'snapshot'. Trigger column name in the snapshot relation. diff --git a/macros/tables/oracle/sat_v0.yml b/macros/tables/oracle/sat_v0.yml new file mode 100644 index 00000000..93734999 --- /dev/null +++ b/macros/tables/oracle/sat_v0.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: oracle__sat_v0 + description: > + Oracle-specific adapter implementation of the sat_v0 macro. Called automatically via adapter.dispatch() + when running on Oracle. Uses a lag_source_data CTE instead of QUALIFY for intra-batch deduplication. + See the parent sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this satellite. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns that form the satellite payload. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Skips lag CTE deduplication. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/oracle/sat_v1.yml b/macros/tables/oracle/sat_v1.yml new file mode 100644 index 00000000..067d2abb --- /dev/null +++ b/macros/tables/oracle/sat_v1.yml @@ -0,0 +1,32 @@ +version: 2 + +macros: + - name: oracle__sat_v1 + description: > + Oracle-specific adapter implementation of the sat_v1 macro. Called automatically via adapter.dispatch() + when running on Oracle. See the parent sat_v1 macro for full parameter documentation. + arguments: + - name: sat_v0 + type: string + description: Name of the underlying version 0 satellite. + - name: hashkey + type: string + description: Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: Name of the hashdiff column inside the version 0 satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag derived from the ledts column. Optional, default False. + - name: include_payload + type: boolean + description: Whether to include the payload columns from sat v0. Optional, default True. diff --git a/macros/tables/pit.md b/macros/tables/pit.md new file mode 100644 index 00000000..60975b6c --- /dev/null +++ b/macros/tables/pit.md @@ -0,0 +1,37 @@ +{% docs pit %} + +## Point-in-Time (PIT) Table + +Creates a PIT table to gather snapshot-based information about one hub and its surrounding satellites. +For this macro to work, a snapshot table is required that has a trigger column to identify which snapshots +to include in the PIT table. The easiest way to create such a snapshot table is to use the control_snap macros +provided by this package. + +Features: +- Tracks the active satellite entries for each hub entry at each snapshot. +- Strongly improves performance if upstream queries require many JOIN operations. +- Creates a unique dimension key to optimize loading performance of incremental loads. +- Allows inserting a static string as record source column, matching business vault definition. + +### Usage + +```jinja +{{ datavault4dbt.pit( + tracked_entity='account_h', + hashkey='hk_account_h', + sat_names=[ + 'account_data_sfdc_1_s', + 'account_financials_erp_1_s' + ], + snapshot_relation='control_snap_v1', + dimension_key='hk_account_h_d' +) }} +``` + +### Metadata block usage + +```jinja +{{ datavault4dbt.pit(yaml_metadata=meta) }} +``` + +{% enddocs %} diff --git a/macros/tables/pit.yml b/macros/tables/pit.yml new file mode 100644 index 00000000..2901300f --- /dev/null +++ b/macros/tables/pit.yml @@ -0,0 +1,79 @@ +version: 2 + +macros: + - name: pit + description: '{{ doc("pit") }}' + arguments: + - name: yaml_metadata + type: dictionary + description: > + Optional. A YAML metadata block that passes all parameters as a single dictionary instead + of specifying each one directly in the macro call. + - name: tracked_entity + type: string + description: > + Name of the tracked Hub entity. Must be available as a model inside the dbt project. + - name: hashkey + type: string + description: > + The name of the hashkey column inside the previously referred Hub entity. + - name: sat_names + type: list of strings + description: > + A list of all the satellites that should be included in this PIT table. Can only be satellites that + are attached to the tracked Hub, and should typically include all those satellites. You should always + refer here to the version 1 satellites, since those hold the load-end-date. The macro currently + supports regular satellites and nh-satellites. + - name: snapshot_relation + type: string + description: > + The name of the snapshot relation. It needs to be available as a model inside this dbt project. + - name: dimension_key + type: string + description: > + The desired name of the dimension key inside the PIT table. Should follow some naming conventions. + Recommended is the name of the hashkey with a '_d' suffix. + - name: snapshot_trigger_column + type: string + description: > + The name of the column inside the previously mentioned snapshot relation, that is boolean and identifies + the snapshots that should be included in the PIT table. + - name: ldts + type: string + description: > + Name of the ldts column inside all source models. Is optional, will use the global variable + 'datavault4dbt.ldts_alias'. Needs to use the same column name as defined as alias inside the staging model. + - name: custom_rsrc + type: string + description: > + A custom string that should be inserted into the 'rsrc' column inside the PIT table. Since a PIT table + is a business vault entity, the technical record source is no longer used here. Is optional, if not + defined, no column is added. + - name: ledts + type: string + description: > + Name of the load-end-date column inside the satellites. Is optional, will use the global variable + 'datavault4dbt.ledts_alias' if not set here. + - name: sdts + type: string + description: > + Name of the snapshot date timestamp column inside the snapshot table. It is optional, will use the + global variable 'datavault4dbt.sdts_alias' if not set here. + - name: pit_type + type: string + description: > + String to insert into the 'pit_type' column. Has to be prefixed by a !. Can be set freely, something + like 'PIT' could be the default. Is optional, if not set, no column will be added. + - name: refer_to_ghost_records + type: boolean + description: > + Value to define if a NULL satellite hashkey should be replaced by the unknown key. + Optional parameter, default is True. + - name: snapshot_optimization + type: boolean + description: > + If set to True, and if the model is run in incremental mode, only the relevant snapshots + (i.e. those that are newer than or equal to the max sdts in the existing PIT table) will be considered. + This can significantly improve performance of incremental loads on large snapshot tables. If set to True, + the model needs to be configured with a unique_key constraint as there may be updates due to late arriving data. + Affected Adapters: Snowflake only! Optional parameter, default is False. diff --git a/macros/tables/postgres/control_snap_v0.yml b/macros/tables/postgres/control_snap_v0.yml new file mode 100644 index 00000000..46fa93ff --- /dev/null +++ b/macros/tables/postgres/control_snap_v0.yml @@ -0,0 +1,21 @@ +version: 2 + +macros: + - name: postgres__control_snap_v0 + description: > + PostgreSQL-specific adapter implementation of the control_snap_v0 macro. Called automatically via + adapter.dispatch() when running on PostgreSQL. See the parent control_snap_v0 macro for full parameter + documentation. + arguments: + - name: start_date + type: timestamp + description: Earliest timestamp for the snapshot table. Time part must be '00:00:00'. + - name: daily_snapshot_time + type: time + description: Time that daily snapshots should have (e.g. '07:00:00'). + - name: sdts_alias + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: end_date + type: timestamp + description: Latest timestamp for the snapshot table. Optional. diff --git a/macros/tables/postgres/control_snap_v1.yml b/macros/tables/postgres/control_snap_v1.yml new file mode 100644 index 00000000..f8c705d7 --- /dev/null +++ b/macros/tables/postgres/control_snap_v1.yml @@ -0,0 +1,20 @@ +version: 2 + +macros: + - name: postgres__control_snap_v1 + description: > + PostgreSQL-specific adapter implementation of the control_snap_v1 macro. Called automatically via + adapter.dispatch() when running on PostgreSQL. See the parent control_snap_v1 macro for full parameter + documentation. + arguments: + - name: control_snap_v0 + type: string + description: The name of the underlying version 0 control snapshot table. + - name: log_logic + type: dictionary + description: > + Defining the desired durations of each granularity ('daily', 'weekly', 'monthly', 'yearly'). + Optional — if not set, all snapshots remain active. + - name: sdts_alias + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. diff --git a/macros/tables/postgres/eff_sat_v0.yml b/macros/tables/postgres/eff_sat_v0.yml new file mode 100644 index 00000000..c2cfe632 --- /dev/null +++ b/macros/tables/postgres/eff_sat_v0.yml @@ -0,0 +1,33 @@ +version: 2 + +macros: + - name: postgres__eff_sat_v0 + description: > + PostgreSQL-specific adapter implementation of the eff_sat_v0 macro. Called automatically via + adapter.dispatch() when running on PostgreSQL. + See the parent eff_sat_v0 macro for full parameter documentation. + arguments: + - name: tracked_hashkey + type: string + description: Name of the hashkey column to be tracked. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: is_active_alias + type: string + description: Name of the active flag column. Optional, defaults to 'datavault4dbt.is_active_alias'. + - name: source_model + type: string + description: Name of the source model. + - name: source_is_single_batch + type: boolean + description: REQUIRED. Whether the source contains only one batch. Must be explicitly set. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be disabled. Optional. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/postgres/hub.yml b/macros/tables/postgres/hub.yml new file mode 100644 index 00000000..04858a7a --- /dev/null +++ b/macros/tables/postgres/hub.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: postgres__hub + description: > + PostgreSQL-specific adapter implementation of the hub macro. Called automatically via adapter.dispatch() + when running on PostgreSQL. See the parent hub macro for full parameter documentation. + arguments: + - name: hashkey + type: string + description: Name of the hashkey column inside the stage, used as PK of the Hub. + - name: business_keys + type: string | list of strings + description: Name(s) of the business key columns loaded into the hub. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include in the Hub. Optional, defaults to empty list. diff --git a/macros/tables/postgres/link.yml b/macros/tables/postgres/link.yml new file mode 100644 index 00000000..5a5b9ece --- /dev/null +++ b/macros/tables/postgres/link.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: postgres__link + description: > + PostgreSQL-specific adapter implementation of the link macro. Called automatically via adapter.dispatch() + when running on PostgreSQL. See the parent link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: Name of the link hashkey column inside the stage. + - name: foreign_hashkeys + type: list of strings + description: List of all hashkey columns inside the link that refer to other hub entities. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include in the Link. Optional, defaults to empty list. diff --git a/macros/tables/postgres/ma_sat_v0.yml b/macros/tables/postgres/ma_sat_v0.yml new file mode 100644 index 00000000..35808900 --- /dev/null +++ b/macros/tables/postgres/ma_sat_v0.yml @@ -0,0 +1,39 @@ +version: 2 + +macros: + - name: postgres__ma_sat_v0 + description: > + PostgreSQL-specific adapter implementation of the ma_sat_v0 macro. Called automatically via + adapter.dispatch() when running on PostgreSQL. Uses a subquery-based approach for intra-batch + deduplication. See the parent ma_sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this satellite. + - name: src_ma_key + type: string | list of strings + description: Name(s) of the multi-active key column(s). + - name: src_payload + type: list of strings + description: List of descriptive attribute columns (excluding the multi-active key). + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Skips deduplication subquery. Optional, default False. diff --git a/macros/tables/postgres/ma_sat_v1.yml b/macros/tables/postgres/ma_sat_v1.yml new file mode 100644 index 00000000..2f5ed0e9 --- /dev/null +++ b/macros/tables/postgres/ma_sat_v1.yml @@ -0,0 +1,33 @@ +version: 2 + +macros: + - name: postgres__ma_sat_v1 + description: > + PostgreSQL-specific adapter implementation of the ma_sat_v1 macro. Called automatically via + adapter.dispatch() when running on PostgreSQL. See the parent ma_sat_v1 macro for full parameter + documentation. + arguments: + - name: sat_v0 + type: string + description: Name of the underlying version 0 multi-active satellite. + - name: hashkey + type: string + description: Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: Name of the hashdiff column inside the version 0 satellite. + - name: ma_attribute + type: string | list of strings + description: Name of the multi-active attribute(s) in the v0 satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag derived from the ledts column. Optional, default False. diff --git a/macros/tables/postgres/nh_link.yml b/macros/tables/postgres/nh_link.yml new file mode 100644 index 00000000..1a7cfd82 --- /dev/null +++ b/macros/tables/postgres/nh_link.yml @@ -0,0 +1,38 @@ +version: 2 + +macros: + - name: postgres__nh_link + description: > + PostgreSQL-specific adapter implementation of the nh_link macro. Called automatically via adapter.dispatch() + when running on PostgreSQL. See the parent nh_link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: Name of the non-historized link hashkey column inside the stage. + - name: payload + type: list of strings + description: List of descriptive attribute columns forming the payload. + - name: foreign_hashkeys + type: list of strings + description: List of all hashkey columns inside the link that refer to other hub entities. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: union_strategy + type: "'all' | 'distinct'" + description: Defines how multiple sources should be unioned. Default 'all'. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/postgres/nh_sat.yml b/macros/tables/postgres/nh_sat.yml new file mode 100644 index 00000000..c3a1c43f --- /dev/null +++ b/macros/tables/postgres/nh_sat.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: postgres__nh_sat + description: > + PostgreSQL-specific adapter implementation of the nh_sat macro. Called automatically via adapter.dispatch() + when running on PostgreSQL. See the parent nh_sat macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns for this non-historized satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/postgres/pit.yml b/macros/tables/postgres/pit.yml new file mode 100644 index 00000000..9b742617 --- /dev/null +++ b/macros/tables/postgres/pit.yml @@ -0,0 +1,47 @@ +version: 2 + +macros: + - name: postgres__pit + description: > + PostgreSQL-specific adapter implementation of the pit macro. Called automatically via adapter.dispatch() + when running on PostgreSQL. See the parent pit macro for full parameter documentation. + arguments: + - name: pit_type + type: string + description: String to insert into the 'pit_type' column, prefixed by !. Optional. + - name: tracked_entity + type: string + description: Name of the tracked Hub entity. + - name: hashkey + type: string + description: Name of the hashkey column inside the tracked Hub. + - name: sat_names + type: list of strings + description: List of all satellite names to include in this PIT table. + - name: ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: sdts + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: custom_rsrc + type: string + description: Custom string to insert into the rsrc column. Optional. + - name: ledts + type: string + description: Name of the load-end-date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: snapshot_relation + type: string + description: Name of the snapshot relation model. + - name: snapshot_trigger_column + type: string + description: Name of the boolean column that identifies active snapshots. + - name: dimension_key + type: string + description: Desired name of the dimension key inside the PIT table. + - name: refer_to_ghost_records + type: boolean + description: Whether a NULL satellite hashkey should be replaced by the unknown key. Optional, default True. + - name: snapshot_optimization + type: boolean + description: Whether to apply snapshot optimization logic. Optional, default False. diff --git a/macros/tables/postgres/rec_track_sat.yml b/macros/tables/postgres/rec_track_sat.yml new file mode 100644 index 00000000..d561bf1e --- /dev/null +++ b/macros/tables/postgres/rec_track_sat.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: postgres__rec_track_sat + description: > + PostgreSQL-specific adapter implementation of the rec_track_sat macro. Called automatically via + adapter.dispatch() when running on PostgreSQL. See the parent rec_track_sat macro for full parameter + documentation. + arguments: + - name: tracked_hashkey + type: string + description: The name of the hashkey column you want to track. + - name: source_models + type: dictionary + description: Dictionary with source model information, including 'rsrc_static' and optionally 'hk_column'. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: src_stg + type: string + description: Name of the source stage model. Optional, defaults to 'datavault4dbt.stg_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/postgres/ref_hub.yml b/macros/tables/postgres/ref_hub.yml new file mode 100644 index 00000000..c4bb09f8 --- /dev/null +++ b/macros/tables/postgres/ref_hub.yml @@ -0,0 +1,23 @@ +version: 2 + +macros: + - name: postgres__ref_hub + description: > + PostgreSQL-specific adapter implementation of the ref_hub macro. Called automatically via adapter.dispatch() + when running on PostgreSQL. See the parent ref_hub macro for full parameter documentation. + arguments: + - name: ref_keys + type: string | list of strings + description: Name of the reference key(s) available in the source model(s). + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/postgres/ref_sat_v0.yml b/macros/tables/postgres/ref_sat_v0.yml new file mode 100644 index 00000000..c654f0f2 --- /dev/null +++ b/macros/tables/postgres/ref_sat_v0.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: postgres__ref_sat_v0 + description: > + PostgreSQL-specific adapter implementation of the ref_sat_v0 macro. Called automatically via + adapter.dispatch() when running on PostgreSQL. Uses a subquery-based approach for deduplication. + See the parent ref_sat_v0 macro for full parameter documentation. + arguments: + - name: parent_ref_keys + type: string | list of strings + description: Name of the reference key(s) of the parent ref_hub. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this reference satellite. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/postgres/ref_sat_v1.yml b/macros/tables/postgres/ref_sat_v1.yml new file mode 100644 index 00000000..c02fdbc3 --- /dev/null +++ b/macros/tables/postgres/ref_sat_v1.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: postgres__ref_sat_v1 + description: > + PostgreSQL-specific adapter implementation of the ref_sat_v1 macro. Called automatically via + adapter.dispatch() when running on PostgreSQL. See the parent ref_sat_v1 macro for full parameter + documentation. + arguments: + - name: ref_sat_v0 + type: string + description: Name of the underlying ref_sat_v0 dbt model. + - name: ref_keys + type: string | list of strings + description: Name(s) of the reference key(s) in the underlying reference sat v0. + - name: hashdiff + type: string + description: Name of the Hashdiff column in the underlying reference sat v0. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag. Optional, default False. diff --git a/macros/tables/postgres/ref_table.yml b/macros/tables/postgres/ref_table.yml new file mode 100644 index 00000000..aed2d031 --- /dev/null +++ b/macros/tables/postgres/ref_table.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: postgres__ref_table + description: > + PostgreSQL-specific adapter implementation of the ref_table macro. Called automatically via adapter.dispatch() + when running on PostgreSQL. See the parent ref_table macro for full parameter documentation. + arguments: + - name: ref_hub + type: string + description: Name of the underlying ref_hub model. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ref_satellites + type: string | list of strings + description: Name(s) of the reference satellites to include. + - name: historized + type: string + description: Possible values are 'full', 'latest', or 'snapshot'. + - name: snapshot_relation + type: string + description: Only required if historized is 'snapshot'. Name of the snapshot_v1 model. + - name: snapshot_trigger_column + type: string + description: Only required if historized is 'snapshot'. Trigger column name in the snapshot relation. diff --git a/macros/tables/postgres/sat_v0.yml b/macros/tables/postgres/sat_v0.yml new file mode 100644 index 00000000..bd498f64 --- /dev/null +++ b/macros/tables/postgres/sat_v0.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: postgres__sat_v0 + description: > + PostgreSQL-specific adapter implementation of the sat_v0 macro. Called automatically via adapter.dispatch() + when running on PostgreSQL. Uses a subquery-based approach for intra-batch deduplication. + See the parent sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this satellite. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns that form the satellite payload. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Skips deduplication subquery. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/postgres/sat_v1.yml b/macros/tables/postgres/sat_v1.yml new file mode 100644 index 00000000..d063b946 --- /dev/null +++ b/macros/tables/postgres/sat_v1.yml @@ -0,0 +1,32 @@ +version: 2 + +macros: + - name: postgres__sat_v1 + description: > + PostgreSQL-specific adapter implementation of the sat_v1 macro. Called automatically via adapter.dispatch() + when running on PostgreSQL. See the parent sat_v1 macro for full parameter documentation. + arguments: + - name: sat_v0 + type: string + description: Name of the underlying version 0 satellite. + - name: hashkey + type: string + description: Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: Name of the hashdiff column inside the version 0 satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag derived from the ledts column. Optional, default False. + - name: include_payload + type: boolean + description: Whether to include the payload columns from sat v0. Optional, default True. diff --git a/macros/tables/rec_track_sat.md b/macros/tables/rec_track_sat.md new file mode 100644 index 00000000..ab172174 --- /dev/null +++ b/macros/tables/rec_track_sat.md @@ -0,0 +1,40 @@ +{% docs rec_track_sat %} + +## Record Tracking Satellite + +Creates a Record Tracking Satellite, most commonly used to track the appearances of hashkeys (calculated out of +business keys) inside one or multiple source systems. This can either be the hashkey of a hub, or the hashkey +of a link. Typically if a hub is loaded from three sources, the corresponding Record Tracking Satellite would +track the same three sources. + +Features: +- Tracks the appearance of a specific hashkey in one or more staging areas. +- Allows source mappings for deviations between the hashkey name inside the stages and the target. +- Supports multiple updates per batch and therefore initial loading. +- Uses a dynamic high-water-mark to optimize loading performance of multiple loads. +- Can track either link or hub hashkeys. + +### Usage + +```jinja +{{ datavault4dbt.rec_track_sat( + tracked_hashkey='hk_contact_h', + source_models={ + 'stg_contact_crm': { + 'rsrc_static': '*/CRM/Contact/*' + }, + 'stg_contact_erp': { + 'hk_column': 'hk_contact_erp_h', + 'rsrc_static': '*/ERP/Contact/*' + } + } +) }} +``` + +### Metadata block usage + +```jinja +{{ datavault4dbt.rec_track_sat(yaml_metadata=meta) }} +``` + +{% enddocs %} diff --git a/macros/tables/rec_track_sat.yml b/macros/tables/rec_track_sat.yml new file mode 100644 index 00000000..40caab32 --- /dev/null +++ b/macros/tables/rec_track_sat.yml @@ -0,0 +1,50 @@ +version: 2 + +macros: + - name: rec_track_sat + description: '{{ doc("rec_track_sat") }}' + arguments: + - name: yaml_metadata + type: dictionary + description: > + Optional. A YAML metadata block that passes all parameters as a single dictionary instead + of specifying each one directly in the macro call. + - name: tracked_hashkey + type: string + description: > + The name of the hashkey column you want to track. Needs to be available in the underlying staging layer. + If you want to track multiple hashkeys out of one stage, you need to create one record tracking satellite + for each hashkey. + Examples: 'hk_contact_h' (tracks appearance of the contact hub hashkey), + 'hk_contact_account_l' (tracks appearance of the link hashkey). + - name: source_models + type: dictionary + description: > + Dictionary with information about the source model. The key of the dict is the name of the source model, + and the value is another dictionary. This inner dictionary requires to have the key 'rsrc_static', and + optionally the key 'hk_column'. The 'rsrc_static' attribute defines a STRING or list of strings that will + always be the same over all loads of one source. Wildcards (e.g. '*') can be used to match variable parts. + If rsrc_static is solely '*', the performance lookup will not be executed. + - name: src_ldts + type: string + description: > + Name of the ldts column inside the source models. Is optional, will use the global variable 'datavault4dbt.ldts_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: src_rsrc + type: string + description: > + Name of the rsrc column inside the source models. Is optional, will use the global variable 'datavault4dbt.rsrc_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: src_stg + type: string + description: > + Name of the source stage model. Is optional, will use the global variable 'datavault4dbt.stg_alias'. + - name: disable_hwm + type: boolean + description: > + Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: > + Additional columns from source system or derived columns which should be part of the rec_track_sat. + The columns need to be available in all source models. Optional parameter, defaults to empty list. diff --git a/macros/tables/redshift/control_snap_v0.yml b/macros/tables/redshift/control_snap_v0.yml new file mode 100644 index 00000000..1a2efc24 --- /dev/null +++ b/macros/tables/redshift/control_snap_v0.yml @@ -0,0 +1,21 @@ +version: 2 + +macros: + - name: redshift__control_snap_v0 + description: > + Redshift-specific adapter implementation of the control_snap_v0 macro. Called automatically via + adapter.dispatch() when running on Redshift. See the parent control_snap_v0 macro for full parameter + documentation. + arguments: + - name: start_date + type: timestamp + description: Earliest timestamp for the snapshot table. Time part must be '00:00:00'. + - name: daily_snapshot_time + type: time + description: Time that daily snapshots should have (e.g. '07:00:00'). + - name: sdts_alias + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: end_date + type: timestamp + description: Latest timestamp for the snapshot table. Optional. diff --git a/macros/tables/redshift/control_snap_v1.yml b/macros/tables/redshift/control_snap_v1.yml new file mode 100644 index 00000000..49b4c433 --- /dev/null +++ b/macros/tables/redshift/control_snap_v1.yml @@ -0,0 +1,20 @@ +version: 2 + +macros: + - name: redshift__control_snap_v1 + description: > + Redshift-specific adapter implementation of the control_snap_v1 macro. Called automatically via + adapter.dispatch() when running on Redshift. See the parent control_snap_v1 macro for full parameter + documentation. + arguments: + - name: control_snap_v0 + type: string + description: The name of the underlying version 0 control snapshot table. + - name: log_logic + type: dictionary + description: > + Defining the desired durations of each granularity ('daily', 'weekly', 'monthly', 'yearly'). + Optional — if not set, all snapshots remain active. + - name: sdts_alias + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. diff --git a/macros/tables/redshift/eff_sat_v0.yml b/macros/tables/redshift/eff_sat_v0.yml new file mode 100644 index 00000000..0652105d --- /dev/null +++ b/macros/tables/redshift/eff_sat_v0.yml @@ -0,0 +1,33 @@ +version: 2 + +macros: + - name: redshift__eff_sat_v0 + description: > + Redshift-specific adapter implementation of the eff_sat_v0 macro. Called automatically via adapter.dispatch() + when running on Redshift. Uses QUALIFY for deduplication. + See the parent eff_sat_v0 macro for full parameter documentation. + arguments: + - name: tracked_hashkey + type: string + description: Name of the hashkey column to be tracked. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: is_active_alias + type: string + description: Name of the active flag column. Optional, defaults to 'datavault4dbt.is_active_alias'. + - name: source_model + type: string + description: Name of the source model. + - name: source_is_single_batch + type: boolean + description: REQUIRED. Whether the source contains only one batch. Must be explicitly set. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be disabled. Optional. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/redshift/hub.yml b/macros/tables/redshift/hub.yml new file mode 100644 index 00000000..af348214 --- /dev/null +++ b/macros/tables/redshift/hub.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: redshift__hub + description: > + Redshift-specific adapter implementation of the hub macro. Called automatically via adapter.dispatch() + when running on Redshift. Uses QUALIFY for deduplication. + See the parent hub macro for full parameter documentation. + arguments: + - name: hashkey + type: string + description: Name of the hashkey column inside the stage, used as PK of the Hub. + - name: business_keys + type: string | list of strings + description: Name(s) of the business key columns loaded into the hub. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include in the Hub. Optional, defaults to empty list. diff --git a/macros/tables/redshift/link.yml b/macros/tables/redshift/link.yml new file mode 100644 index 00000000..dcddda53 --- /dev/null +++ b/macros/tables/redshift/link.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: redshift__link + description: > + Redshift-specific adapter implementation of the link macro. Called automatically via adapter.dispatch() + when running on Redshift. Uses QUALIFY for deduplication. + See the parent link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: Name of the link hashkey column inside the stage. + - name: foreign_hashkeys + type: list of strings + description: List of all hashkey columns inside the link that refer to other hub entities. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include in the Link. Optional, defaults to empty list. diff --git a/macros/tables/redshift/ma_sat_v0.yml b/macros/tables/redshift/ma_sat_v0.yml new file mode 100644 index 00000000..5bf58e78 --- /dev/null +++ b/macros/tables/redshift/ma_sat_v0.yml @@ -0,0 +1,39 @@ +version: 2 + +macros: + - name: redshift__ma_sat_v0 + description: > + Redshift-specific adapter implementation of the ma_sat_v0 macro. Called automatically via adapter.dispatch() + when running on Redshift. Uses QUALIFY for intra-batch deduplication. + See the parent ma_sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this satellite. + - name: src_ma_key + type: string | list of strings + description: Name(s) of the multi-active key column(s). + - name: src_payload + type: list of strings + description: List of descriptive attribute columns (excluding the multi-active key). + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Skips QUALIFY deduplication. Optional, default False. diff --git a/macros/tables/redshift/ma_sat_v1.yml b/macros/tables/redshift/ma_sat_v1.yml new file mode 100644 index 00000000..08c545ff --- /dev/null +++ b/macros/tables/redshift/ma_sat_v1.yml @@ -0,0 +1,32 @@ +version: 2 + +macros: + - name: redshift__ma_sat_v1 + description: > + Redshift-specific adapter implementation of the ma_sat_v1 macro. Called automatically via adapter.dispatch() + when running on Redshift. See the parent ma_sat_v1 macro for full parameter documentation. + arguments: + - name: sat_v0 + type: string + description: Name of the underlying version 0 multi-active satellite. + - name: hashkey + type: string + description: Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: Name of the hashdiff column inside the version 0 satellite. + - name: ma_attribute + type: string | list of strings + description: Name of the multi-active attribute(s) in the v0 satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag derived from the ledts column. Optional, default False. diff --git a/macros/tables/redshift/nh_link.yml b/macros/tables/redshift/nh_link.yml new file mode 100644 index 00000000..f64433e6 --- /dev/null +++ b/macros/tables/redshift/nh_link.yml @@ -0,0 +1,38 @@ +version: 2 + +macros: + - name: redshift__nh_link + description: > + Redshift-specific adapter implementation of the nh_link macro. Called automatically via adapter.dispatch() + when running on Redshift. See the parent nh_link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: Name of the non-historized link hashkey column inside the stage. + - name: payload + type: list of strings + description: List of descriptive attribute columns forming the payload. + - name: foreign_hashkeys + type: list of strings + description: List of all hashkey columns inside the link that refer to other hub entities. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: union_strategy + type: "'all' | 'distinct'" + description: Defines how multiple sources should be unioned. Default 'all'. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/redshift/nh_sat.yml b/macros/tables/redshift/nh_sat.yml new file mode 100644 index 00000000..2894ffb1 --- /dev/null +++ b/macros/tables/redshift/nh_sat.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: redshift__nh_sat + description: > + Redshift-specific adapter implementation of the nh_sat macro. Called automatically via adapter.dispatch() + when running on Redshift. See the parent nh_sat macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns for this non-historized satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/redshift/pit.yml b/macros/tables/redshift/pit.yml new file mode 100644 index 00000000..a3de5b30 --- /dev/null +++ b/macros/tables/redshift/pit.yml @@ -0,0 +1,47 @@ +version: 2 + +macros: + - name: redshift__pit + description: > + Redshift-specific adapter implementation of the pit macro. Called automatically via adapter.dispatch() + when running on Redshift. See the parent pit macro for full parameter documentation. + arguments: + - name: pit_type + type: string + description: String to insert into the 'pit_type' column, prefixed by !. Optional. + - name: tracked_entity + type: string + description: Name of the tracked Hub entity. + - name: hashkey + type: string + description: Name of the hashkey column inside the tracked Hub. + - name: sat_names + type: list of strings + description: List of all satellite names to include in this PIT table. + - name: ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: sdts + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: custom_rsrc + type: string + description: Custom string to insert into the rsrc column. Optional. + - name: ledts + type: string + description: Name of the load-end-date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: snapshot_relation + type: string + description: Name of the snapshot relation model. + - name: snapshot_trigger_column + type: string + description: Name of the boolean column that identifies active snapshots. + - name: dimension_key + type: string + description: Desired name of the dimension key inside the PIT table. + - name: refer_to_ghost_records + type: boolean + description: Whether a NULL satellite hashkey should be replaced by the unknown key. Optional, default True. + - name: snapshot_optimization + type: boolean + description: Whether to apply snapshot optimization logic. Optional, default False. diff --git a/macros/tables/redshift/rec_track_sat.yml b/macros/tables/redshift/rec_track_sat.yml new file mode 100644 index 00000000..4f684bcd --- /dev/null +++ b/macros/tables/redshift/rec_track_sat.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: redshift__rec_track_sat + description: > + Redshift-specific adapter implementation of the rec_track_sat macro. Called automatically via + adapter.dispatch() when running on Redshift. See the parent rec_track_sat macro for full parameter + documentation. + arguments: + - name: tracked_hashkey + type: string + description: The name of the hashkey column you want to track. + - name: source_models + type: dictionary + description: Dictionary with source model information, including 'rsrc_static' and optionally 'hk_column'. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: src_stg + type: string + description: Name of the source stage model. Optional, defaults to 'datavault4dbt.stg_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/redshift/ref_hub.yml b/macros/tables/redshift/ref_hub.yml new file mode 100644 index 00000000..8402ff29 --- /dev/null +++ b/macros/tables/redshift/ref_hub.yml @@ -0,0 +1,23 @@ +version: 2 + +macros: + - name: redshift__ref_hub + description: > + Redshift-specific adapter implementation of the ref_hub macro. Called automatically via adapter.dispatch() + when running on Redshift. See the parent ref_hub macro for full parameter documentation. + arguments: + - name: ref_keys + type: string | list of strings + description: Name of the reference key(s) available in the source model(s). + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/redshift/ref_sat_v0.yml b/macros/tables/redshift/ref_sat_v0.yml new file mode 100644 index 00000000..13e04eae --- /dev/null +++ b/macros/tables/redshift/ref_sat_v0.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: redshift__ref_sat_v0 + description: > + Redshift-specific adapter implementation of the ref_sat_v0 macro. Called automatically via adapter.dispatch() + when running on Redshift. Uses QUALIFY for deduplication. + See the parent ref_sat_v0 macro for full parameter documentation. + arguments: + - name: parent_ref_keys + type: string | list of strings + description: Name of the reference key(s) of the parent ref_hub. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this reference satellite. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/redshift/ref_sat_v1.yml b/macros/tables/redshift/ref_sat_v1.yml new file mode 100644 index 00000000..9eb100a5 --- /dev/null +++ b/macros/tables/redshift/ref_sat_v1.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: redshift__ref_sat_v1 + description: > + Redshift-specific adapter implementation of the ref_sat_v1 macro. Called automatically via adapter.dispatch() + when running on Redshift. See the parent ref_sat_v1 macro for full parameter documentation. + arguments: + - name: ref_sat_v0 + type: string + description: Name of the underlying ref_sat_v0 dbt model. + - name: ref_keys + type: string | list of strings + description: Name(s) of the reference key(s) in the underlying reference sat v0. + - name: hashdiff + type: string + description: Name of the Hashdiff column in the underlying reference sat v0. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag. Optional, default False. diff --git a/macros/tables/redshift/ref_table.yml b/macros/tables/redshift/ref_table.yml new file mode 100644 index 00000000..d1e24246 --- /dev/null +++ b/macros/tables/redshift/ref_table.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: redshift__ref_table + description: > + Redshift-specific adapter implementation of the ref_table macro. Called automatically via adapter.dispatch() + when running on Redshift. See the parent ref_table macro for full parameter documentation. + arguments: + - name: ref_hub + type: string + description: Name of the underlying ref_hub model. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ref_satellites + type: string | list of strings + description: Name(s) of the reference satellites to include. + - name: historized + type: string + description: Possible values are 'full', 'latest', or 'snapshot'. + - name: snapshot_relation + type: string + description: Only required if historized is 'snapshot'. Name of the snapshot_v1 model. + - name: snapshot_trigger_column + type: string + description: Only required if historized is 'snapshot'. Trigger column name in the snapshot relation. diff --git a/macros/tables/redshift/sat_v0.yml b/macros/tables/redshift/sat_v0.yml new file mode 100644 index 00000000..4930c096 --- /dev/null +++ b/macros/tables/redshift/sat_v0.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: redshift__sat_v0 + description: > + Redshift-specific adapter implementation of the sat_v0 macro. Called automatically via adapter.dispatch() + when running on Redshift. Uses QUALIFY for intra-batch deduplication. + See the parent sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this satellite. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns that form the satellite payload. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Skips QUALIFY deduplication. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/redshift/sat_v1.yml b/macros/tables/redshift/sat_v1.yml new file mode 100644 index 00000000..1a2d043a --- /dev/null +++ b/macros/tables/redshift/sat_v1.yml @@ -0,0 +1,32 @@ +version: 2 + +macros: + - name: redshift__sat_v1 + description: > + Redshift-specific adapter implementation of the sat_v1 macro. Called automatically via adapter.dispatch() + when running on Redshift. See the parent sat_v1 macro for full parameter documentation. + arguments: + - name: sat_v0 + type: string + description: Name of the underlying version 0 satellite. + - name: hashkey + type: string + description: Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: Name of the hashdiff column inside the version 0 satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag derived from the ledts column. Optional, default False. + - name: include_payload + type: boolean + description: Whether to include the payload columns from sat v0. Optional, default True. diff --git a/macros/tables/ref_hub.md b/macros/tables/ref_hub.md new file mode 100644 index 00000000..c4ab0ca3 --- /dev/null +++ b/macros/tables/ref_hub.md @@ -0,0 +1,29 @@ +{% docs ref_hub %} + +## Reference Hub + +Creates a Reference Hub entity in the Raw Data Vault. A Reference Hub stores the unique reference keys for +reference data (e.g. country codes, product categories) that does not originate from a business entity +but is used to classify and enrich other entities. + +### Usage + +```jinja +{{ datavault4dbt.ref_hub( + ref_keys='country_code', + source_models={ + 'stg_country': { + 'ref_keys': 'country_code', + 'rsrc_static': '*/REF/Country/*' + } + } +) }} +``` + +### Metadata block usage + +```jinja +{{ datavault4dbt.ref_hub(yaml_metadata=meta) }} +``` + +{% enddocs %} diff --git a/macros/tables/ref_hub.yml b/macros/tables/ref_hub.yml new file mode 100644 index 00000000..1df121fd --- /dev/null +++ b/macros/tables/ref_hub.yml @@ -0,0 +1,34 @@ +version: 2 + +macros: + - name: ref_hub + description: '{{ doc("ref_hub") }}' + arguments: + - name: yaml_metadata + type: dictionary + description: > + Optional. A YAML metadata block that passes all parameters as a single dictionary instead + of specifying each one directly in the macro call. + - name: ref_keys + type: string | list of strings + description: > + Name of the reference key(s) available in the source model(s). + - name: source_models + type: dictionary + description: > + Similar to other source_models parameters, e.g. in Hubs or Links. + - name: src_ldts + type: string + description: > + Name of the ldts column inside the source models. Is optional, will use the global variable 'datavault4dbt.ldts_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: src_rsrc + type: string + description: > + Name of the rsrc column inside the source models. Is optional, will use the global variable 'datavault4dbt.rsrc_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: additional_columns + type: list | string + description: > + Additional columns from source system or derived columns which should be part of the ref_hub. + The columns need to be available in all source models. Optional parameter, defaults to empty list. diff --git a/macros/tables/ref_sat_v0.md b/macros/tables/ref_sat_v0.md new file mode 100644 index 00000000..54bd28e5 --- /dev/null +++ b/macros/tables/ref_sat_v0.md @@ -0,0 +1,26 @@ +{% docs ref_sat_v0 %} + +## Reference Satellite v0 + +Creates a version 0 Reference Satellite in the Raw Data Vault. A Reference Satellite stores the descriptive +attributes (payload) of a reference entity over time, tracking every change. It is linked to its parent +Reference Hub via the reference key. + +### Usage + +```jinja +{{ datavault4dbt.ref_sat_v0( + parent_ref_keys='country_code', + src_hashdiff='hd_country_rs', + src_payload=['country_name', 'continent', 'region'], + source_model='stg_country' +) }} +``` + +### Metadata block usage + +```jinja +{{ datavault4dbt.ref_sat_v0(yaml_metadata=meta) }} +``` + +{% enddocs %} diff --git a/macros/tables/ref_sat_v0.yml b/macros/tables/ref_sat_v0.yml new file mode 100644 index 00000000..2bf09fc7 --- /dev/null +++ b/macros/tables/ref_sat_v0.yml @@ -0,0 +1,56 @@ +version: 2 + +macros: + - name: ref_sat_v0 + description: '{{ doc("ref_sat_v0") }}' + arguments: + - name: yaml_metadata + type: dictionary + description: > + Optional. A YAML metadata block that passes all parameters as a single dictionary instead + of specifying each one directly in the macro call. + - name: parent_ref_keys + type: string | list of strings + description: > + Name of the reference key(s) of the parent ref_hub. + - name: src_hashdiff + type: string + description: > + Name of the hashdiff column of this ref satellite, that was created inside the staging area and is + calculated out of the entire payload of this ref satellite. The stage must hold one hashdiff per + ref satellite entity. + Examples: 'hd_nation_sfdc_rs' — hashdiff column of the ref satellite for nation. + - name: src_payload + type: list of strings + description: > + A list of all the descriptive attributes that should be included in this ref satellite. Needs to be the + columns that are fed into the hashdiff calculation of this ref satellite. + Examples: ['name', 'continent', 'area']. + - name: source_model + type: string + description: > + Name of the underlying staging model, must be available inside dbt as a model. + Examples: 'stage_nation' — This ref satellite is loaded out of the stage for account. + - name: src_ldts + type: string + description: > + Name of the ldts column inside the source models. Is optional, will use the global variable 'datavault4dbt.ldts_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: src_rsrc + type: string + description: > + Name of the rsrc column inside the source models. Is optional, will use the global variable 'datavault4dbt.rsrc_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: disable_hwm + type: boolean + description: > + Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: > + Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: > + Additional columns from source system or derived columns which should be part of the ref_sat. + The columns need to be available in all source models. Optional parameter, defaults to empty list. diff --git a/macros/tables/ref_sat_v1.md b/macros/tables/ref_sat_v1.md new file mode 100644 index 00000000..6b65dc16 --- /dev/null +++ b/macros/tables/ref_sat_v1.md @@ -0,0 +1,25 @@ +{% docs ref_sat_v1 %} + +## Reference Satellite v1 + +Creates a version 1 Reference Satellite view derived from an existing ref_sat_v0. Adds a load end date timestamp +(ledts) column, enabling point-in-time queries without window functions. A version 1 ref satellite should be +materialized as a view. + +### Usage + +```jinja +{{ datavault4dbt.ref_sat_v1( + ref_sat_v0='country_0_rs', + ref_keys='country_code', + hashdiff='hd_country_rs' +) }} +``` + +### Metadata block usage + +```jinja +{{ datavault4dbt.ref_sat_v1(yaml_metadata=meta) }} +``` + +{% enddocs %} diff --git a/macros/tables/ref_sat_v1.yml b/macros/tables/ref_sat_v1.yml new file mode 100644 index 00000000..3d3258e0 --- /dev/null +++ b/macros/tables/ref_sat_v1.yml @@ -0,0 +1,44 @@ +version: 2 + +macros: + - name: ref_sat_v1 + description: '{{ doc("ref_sat_v1") }}' + arguments: + - name: yaml_metadata + type: dictionary + description: > + Optional. A YAML metadata block that passes all parameters as a single dictionary instead + of specifying each one directly in the macro call. + - name: ref_sat_v0 + type: string + description: > + Name of the underlying ref_sat_v0 dbt model. + - name: ref_keys + type: string | list of strings + description: > + Name(s) of the reference key(s) in the underlying reference sat v0. + - name: hashdiff + type: string + description: > + Name of the Hashdiff column in the underlying reference sat v0. + - name: src_ldts + type: string + description: > + Name of the ldts column inside the source models. Is optional, will use the global variable 'datavault4dbt.ldts_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: src_rsrc + type: string + description: > + Name of the rsrc column inside the source models. Is optional, will use the global variable 'datavault4dbt.rsrc_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: ledts_alias + type: string + description: > + Desired alias for the load end date column. Is optional, will use the global variable 'datavault4dbt.ledts_alias' + if set here. + - name: add_is_current_flag + type: boolean + description: > + Optional parameter to add a new column to the v1 sat based on the load end date timestamp (ledts). Default is false. + If set to true it will add this is_current flag to the v1 sat. For each record this column will be set to true if + the load end date timestamp is equal to the variable end of all times. If not, the record is not current. diff --git a/macros/tables/ref_table.md b/macros/tables/ref_table.md new file mode 100644 index 00000000..4a5009ec --- /dev/null +++ b/macros/tables/ref_table.md @@ -0,0 +1,36 @@ +{% docs ref_table %} + +## Reference Table + +Creates a Reference Table that combines a ref_hub with one or more ref_satellites into a single queryable entity. +The historization mode controls how much history the table retains: 'latest' keeps only the current record, +'full' keeps all historical records, and 'snapshot' aligns with a snapshot table. + +### Usage (latest — current values only) + +```jinja +{{ datavault4dbt.ref_table( + ref_hub='country_rh', + ref_satellites=['country_1_rs'], + historized='latest' +) }} +``` + +### Usage (snapshot-based) + +```jinja +{{ datavault4dbt.ref_table( + ref_hub='country_rh', + ref_satellites=['country_1_rs'], + historized='snapshot', + snapshot_relation='control_snap_v1' +) }} +``` + +### Metadata block usage + +```jinja +{{ datavault4dbt.ref_table(yaml_metadata=meta) }} +``` + +{% enddocs %} diff --git a/macros/tables/ref_table.yml b/macros/tables/ref_table.yml new file mode 100644 index 00000000..02703050 --- /dev/null +++ b/macros/tables/ref_table.yml @@ -0,0 +1,44 @@ +version: 2 + +macros: + - name: ref_table + description: '{{ doc("ref_table") }}' + arguments: + - name: yaml_metadata + type: dictionary + description: > + Optional. A YAML metadata block that passes all parameters as a single dictionary instead + of specifying each one directly in the macro call. + - name: ref_hub + type: string + description: > + Name of the underlying ref_hub model. + - name: ref_satellites + type: string | list of strings + description: > + Name(s) of the reference satellites to be included in this ref_table. Optional: 'include' and 'exclude' + as dictionary keys for each satellite. + - name: src_ldts + type: string + description: > + Name of the ldts column inside the source models. Is optional, will use the global variable 'datavault4dbt.ldts_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: src_rsrc + type: string + description: > + Name of the rsrc column inside the source models. Is optional, will use the global variable 'datavault4dbt.rsrc_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: historized + type: string + description: > + Possible values are 'full', 'latest', or 'snapshot'. Influences how much history this reference table + will hold. + - name: snapshot_relation + type: string + description: > + Only required if 'historized' is set to 'snapshot'. Name of the snapshot_v1 model to be used. + - name: snapshot_trigger_column + type: string + description: > + Only required if 'historized' is set to 'snapshot'. Defaults to global variable + 'datavault4dbt.snapshot_trigger_column'. Only needs to be set if alias deviates from global variable. diff --git a/macros/tables/sat_v0.md b/macros/tables/sat_v0.md new file mode 100644 index 00000000..f92af2bd --- /dev/null +++ b/macros/tables/sat_v0.md @@ -0,0 +1,33 @@ +{% docs sat_v0 %} + +## Satellite v0 + +Creates a standard satellite version 0, materialized as an incremental table. Applied on top of the staging layer, +connected to either a Hub or a Link. Each satellite can only be loaded by one source model, since we typically +recommend a satellite split by source system. + +On top of each version 0 satellite, a version 1 satellite should be created using the sat_v1 macro, which extends +the v0 satellite by a virtually calculated load end date. + +Features: +- Can handle multiple updates per batch, without losing intermediate changes — initial loading is supported. +- Uses a dynamic high-water-mark to optimize loading performance of multiple loads. + +### Usage + +```jinja +{{ datavault4dbt.sat_v0( + parent_hashkey='hk_account_h', + src_hashdiff='hd_account_data_sfdc_s', + src_payload=['name', 'address', 'country', 'phone', 'email'], + source_model='stage_account' +) }} +``` + +### Metadata block usage + +```jinja +{{ datavault4dbt.sat_v0(yaml_metadata=meta) }} +``` + +{% enddocs %} diff --git a/macros/tables/sat_v0.yml b/macros/tables/sat_v0.yml new file mode 100644 index 00000000..466fed55 --- /dev/null +++ b/macros/tables/sat_v0.yml @@ -0,0 +1,56 @@ +version: 2 + +macros: + - name: sat_v0 + description: '{{ doc("sat_v0") }}' + arguments: + - name: yaml_metadata + type: dictionary + description: > + Optional. A YAML metadata block that passes all parameters as a single dictionary instead + of specifying each one directly in the macro call. + - name: parent_hashkey + type: string + description: > + Name of the hashkey column inside the stage of the object that this satellite is attached to. + Examples: 'hk_account_h' (attached to hub account), 'hk_account_contact_l' (attached to a link). + - name: src_hashdiff + type: string + description: > + Name of the hashdiff column of this satellite, that was created inside the staging area and is + calculated out of the entire payload of this satellite. The stage must hold one hashdiff per + satellite entity. Examples: 'hd_account_data_sfdc_s'. + - name: src_payload + type: list of strings + description: > + A list of all the descriptive attributes that should be included in this satellite. Needs to be the + columns that are fed into the hashdiff calculation of this satellite. + Examples: ['name', 'address', 'country', 'phone', 'email']. + - name: source_model + type: string + description: > + Name of the underlying staging model, must be available inside dbt as a model. + Examples: 'stage_account' — This satellite is loaded out of the stage for account. + - name: src_ldts + type: string + description: > + Name of the ldts column inside the source models. Is optional, will use the global variable 'datavault4dbt.ldts_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: src_rsrc + type: string + description: > + Name of the rsrc column inside the source models. Is optional, will use the global variable 'datavault4dbt.rsrc_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: disable_hwm + type: boolean + description: > + Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: > + Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: > + Additional columns from source system or derived columns which should be part of the Sat. + The columns need to be available in all source models. Optional parameter, defaults to empty list. diff --git a/macros/tables/sat_v1.md b/macros/tables/sat_v1.md new file mode 100644 index 00000000..158685d9 --- /dev/null +++ b/macros/tables/sat_v1.md @@ -0,0 +1,36 @@ +{% docs sat_v1 %} + +## Satellite v1 + +Calculates a virtualized load end date on top of a version 0 satellite. This column is generated for usage in the +PIT tables, and only virtualized to follow the insert-only approach. A version 1 satellite should be materialized +as a view by default. Usually one version 1 sat would be created for each version 0 sat. + +### Usage + +```jinja +{{ datavault4dbt.sat_v1( + sat_v0='account_data_sfdc_0_s', + hashkey='hk_account_h', + hashdiff='hd_account_data_sfdc_s' +) }} +``` + +### With is_current flag + +```jinja +{{ datavault4dbt.sat_v1( + sat_v0='account_data_sfdc_0_s', + hashkey='hk_account_h', + hashdiff='hd_account_data_sfdc_s', + add_is_current_flag=true +) }} +``` + +### Metadata block usage + +```jinja +{{ datavault4dbt.sat_v1(yaml_metadata=meta) }} +``` + +{% enddocs %} diff --git a/macros/tables/sat_v1.yml b/macros/tables/sat_v1.yml new file mode 100644 index 00000000..eee4eccf --- /dev/null +++ b/macros/tables/sat_v1.yml @@ -0,0 +1,55 @@ +version: 2 + +macros: + - name: sat_v1 + description: '{{ doc("sat_v1") }}' + arguments: + - name: yaml_metadata + type: dictionary + description: > + Optional. A YAML metadata block that passes all parameters as a single dictionary instead + of specifying each one directly in the macro call. + - name: sat_v0 + type: string + description: > + Name of the underlying version 0 satellite. + Examples: 'account_data_sfdc_0_s' — This satellite would be the version 1 satellite of the underlying + version 0 data satellite for account. + - name: hashkey + type: string + description: > + Name of the parent hashkey column inside the version 0 satellite. Would either be the hashkey of a + hub or a link. Needs to be similar to the 'parent_hashkey' parameter inside the sat_v0 model. + Examples: 'hk_account_h' (hub), 'hk_account_contact_l' (link). + - name: hashdiff + type: string + description: > + Name of the hashdiff column inside the underlying version 0 satellite. Needs to be similar to the + 'src_hashdiff' parameter inside the sat_v0 model. + Examples: 'hd_account_data_sfdc_s'. + - name: src_ldts + type: string + description: > + Name of the ldts column inside the source models. Is optional, will use the global variable 'datavault4dbt.ldts_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: src_rsrc + type: string + description: > + Name of the rsrc column inside the source models. Is optional, will use the global variable 'datavault4dbt.rsrc_alias'. + Needs to use the same column name as defined as alias inside the staging model. + - name: ledts_alias + type: string + description: > + Desired alias for the load end date column. Is optional, will use the global variable 'datavault4dbt.ledts_alias' + if set here. + - name: add_is_current_flag + type: boolean + description: > + Optional parameter to add a new column to the v1 sat based on the load end date timestamp (ledts). Default is false. + If set to true it will add this is_current flag to the v1 sat. For each record this column will be set to true if + the load end date timestamp is equal to the variable end of all times. If not, the record is not current and will + be set to false. + - name: include_payload + type: boolean + description: > + Optional parameter to specify if the v1 sat should have the payload columns from sat v0 or not. Default is true. diff --git a/macros/tables/snowflake/control_snap_v0.yml b/macros/tables/snowflake/control_snap_v0.yml new file mode 100644 index 00000000..e1712c05 --- /dev/null +++ b/macros/tables/snowflake/control_snap_v0.yml @@ -0,0 +1,25 @@ +version: 2 + +macros: + - name: snowflake__control_snap_v0 + description: > + Snowflake-specific adapter implementation of the control_snap_v0 macro. Called automatically via + adapter.dispatch() when running on Snowflake. See the parent control_snap_v0 macro for full parameter + documentation. + arguments: + - name: start_date + type: timestamp + description: > + Earliest timestamp for the snapshot table. Time part must be '00:00:00'. + - name: daily_snapshot_time + type: time + description: > + Time that daily snapshots should have (e.g. '07:00:00'). + - name: sdts_alias + type: string + description: > + Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: end_date + type: timestamp + description: > + Latest timestamp for the snapshot table. Optional. diff --git a/macros/tables/snowflake/control_snap_v1.yml b/macros/tables/snowflake/control_snap_v1.yml new file mode 100644 index 00000000..d3a2a3ab --- /dev/null +++ b/macros/tables/snowflake/control_snap_v1.yml @@ -0,0 +1,22 @@ +version: 2 + +macros: + - name: snowflake__control_snap_v1 + description: > + Snowflake-specific adapter implementation of the control_snap_v1 macro. Called automatically via + adapter.dispatch() when running on Snowflake. See the parent control_snap_v1 macro for full parameter + documentation. + arguments: + - name: control_snap_v0 + type: string + description: > + The name of the underlying version 0 control snapshot table. + - name: log_logic + type: dictionary + description: > + Defining the desired durations of each granularity ('daily', 'weekly', 'monthly', 'yearly'). + Optional — if not set, all snapshots remain active. + - name: sdts_alias + type: string + description: > + Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. diff --git a/macros/tables/snowflake/eff_sat_v0.yml b/macros/tables/snowflake/eff_sat_v0.yml new file mode 100644 index 00000000..016b385c --- /dev/null +++ b/macros/tables/snowflake/eff_sat_v0.yml @@ -0,0 +1,40 @@ +version: 2 + +macros: + - name: snowflake__eff_sat_v0 + description: > + Snowflake-specific adapter implementation of the eff_sat_v0 macro. Called automatically via adapter.dispatch() + when running on Snowflake. See the parent eff_sat_v0 macro for full parameter documentation. + arguments: + - name: tracked_hashkey + type: string + description: > + Name of the hashkey column to be tracked. + - name: src_ldts + type: string + description: > + Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: > + Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: is_active_alias + type: string + description: > + Name of the active flag column. Optional, defaults to 'datavault4dbt.is_active_alias'. + - name: source_model + type: string + description: > + Name of the source model. + - name: source_is_single_batch + type: boolean + description: > + REQUIRED. Whether the source contains only one batch. Must be explicitly set. + - name: disable_hwm + type: boolean + description: > + Whether the High Water Mark should be disabled. Optional. + - name: additional_columns + type: list | string + description: > + Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/snowflake/hub.yml b/macros/tables/snowflake/hub.yml new file mode 100644 index 00000000..bad3af52 --- /dev/null +++ b/macros/tables/snowflake/hub.yml @@ -0,0 +1,38 @@ +version: 2 + +macros: + - name: snowflake__hub + description: > + Snowflake-specific adapter implementation of the hub macro. Called automatically via adapter.dispatch() + when running on Snowflake. Uses a run_query call to pre-fetch the max load date timestamp for more + efficient High Water Mark filtering. See the parent hub macro for full parameter documentation. + arguments: + - name: hashkey + type: string + description: > + Name of the hashkey column inside the stage, that should be used as PK of the Hub. + - name: business_keys + type: string | list of strings + description: > + Name(s) of the business key columns that should be loaded into the hub. + - name: source_models + type: dictionary + description: > + Dictionary with information about the source models, including 'bk_columns', 'hk_column', + and 'rsrc_static' per source. + - name: src_ldts + type: string + description: > + Name of the ldts column inside the source models. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: > + Name of the rsrc column inside the source models. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: > + Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: > + Additional columns to include in the Hub. Optional, defaults to empty list. diff --git a/macros/tables/snowflake/link.yml b/macros/tables/snowflake/link.yml new file mode 100644 index 00000000..901d5eed --- /dev/null +++ b/macros/tables/snowflake/link.yml @@ -0,0 +1,37 @@ +version: 2 + +macros: + - name: snowflake__link + description: > + Snowflake-specific adapter implementation of the link macro. Called automatically via adapter.dispatch() + when running on Snowflake. Uses a run_query call to pre-fetch the max load date timestamp for more + efficient High Water Mark filtering. See the parent link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: > + Name of the link hashkey column inside the stage. + - name: foreign_hashkeys + type: list of strings + description: > + List of all hashkey columns inside the link that refer to other hub entities. + - name: source_models + type: dictionary + description: > + Dictionary with information about the source models. + - name: src_ldts + type: string + description: > + Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: > + Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: > + Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: > + Additional columns to include in the Link. Optional, defaults to empty list. diff --git a/macros/tables/snowflake/ma_sat_v0.yml b/macros/tables/snowflake/ma_sat_v0.yml new file mode 100644 index 00000000..8595f40b --- /dev/null +++ b/macros/tables/snowflake/ma_sat_v0.yml @@ -0,0 +1,49 @@ +version: 2 + +macros: + - name: snowflake__ma_sat_v0 + description: > + Snowflake-specific adapter implementation of the ma_sat_v0 macro. Called automatically via adapter.dispatch() + when running on Snowflake. Uses QUALIFY for intra-batch deduplication and a run_query call for the High + Water Mark. See the parent ma_sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: > + Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: > + Name of the hashdiff column for this satellite. + - name: src_ma_key + type: string | list of strings + description: > + Name(s) of the multi-active key column(s). + - name: src_payload + type: list of strings + description: > + List of descriptive attribute columns (excluding the multi-active key). + - name: src_ldts + type: string + description: > + Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: > + Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: > + Name of the underlying staging model. + - name: additional_columns + type: list | string + description: > + Additional columns to include. Optional, defaults to empty list. + - name: disable_hwm + type: boolean + description: > + Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: > + Whether the source contains only one batch. Skips QUALIFY deduplication. Optional, default False. diff --git a/macros/tables/snowflake/ma_sat_v1.yml b/macros/tables/snowflake/ma_sat_v1.yml new file mode 100644 index 00000000..25d5e36d --- /dev/null +++ b/macros/tables/snowflake/ma_sat_v1.yml @@ -0,0 +1,40 @@ +version: 2 + +macros: + - name: snowflake__ma_sat_v1 + description: > + Snowflake-specific adapter implementation of the ma_sat_v1 macro. Called automatically via adapter.dispatch() + when running on Snowflake. See the parent ma_sat_v1 macro for full parameter documentation. + arguments: + - name: sat_v0 + type: string + description: > + Name of the underlying version 0 multi-active satellite. + - name: hashkey + type: string + description: > + Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: > + Name of the hashdiff column inside the version 0 satellite. + - name: ma_attribute + type: string | list of strings + description: > + Name of the multi-active attribute(s) in the v0 satellite. + - name: src_ldts + type: string + description: > + Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: > + Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: > + Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: > + Whether to add an is_current flag derived from the ledts column. Optional, default False. diff --git a/macros/tables/snowflake/nh_link.yml b/macros/tables/snowflake/nh_link.yml new file mode 100644 index 00000000..7ecc6c0e --- /dev/null +++ b/macros/tables/snowflake/nh_link.yml @@ -0,0 +1,48 @@ +version: 2 + +macros: + - name: snowflake__nh_link + description: > + Snowflake-specific adapter implementation of the nh_link macro. Called automatically via adapter.dispatch() + when running on Snowflake. See the parent nh_link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: > + Name of the non-historized link hashkey column inside the stage. + - name: payload + type: list of strings + description: > + List of descriptive attribute columns that form the payload of this non-historized link. + - name: foreign_hashkeys + type: list of strings + description: > + List of all hashkey columns inside the link that refer to other hub entities. + - name: src_ldts + type: string + description: > + Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: > + Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: > + Dictionary with information about the source models. + - name: disable_hwm + type: boolean + description: > + Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: > + Whether the source contains only one batch. Optional, default False. + - name: union_strategy + type: "'all' | 'distinct'" + description: > + Defines how multiple sources should be unioned. Default 'all'. + - name: additional_columns + type: list | string + description: > + Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/snowflake/nh_sat.yml b/macros/tables/snowflake/nh_sat.yml new file mode 100644 index 00000000..00d9ad5c --- /dev/null +++ b/macros/tables/snowflake/nh_sat.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: snowflake__nh_sat + description: > + Snowflake-specific adapter implementation of the nh_sat macro. Called automatically via adapter.dispatch() + when running on Snowflake. See the parent nh_sat macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: > + Name of the hashkey column of the parent hub or link. + - name: src_payload + type: list of strings + description: > + List of descriptive attribute columns for this non-historized satellite. + - name: src_ldts + type: string + description: > + Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: > + Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: > + Name of the underlying staging model. + - name: source_is_single_batch + type: boolean + description: > + Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: > + Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/snowflake/pit.yml b/macros/tables/snowflake/pit.yml new file mode 100644 index 00000000..24e3ca45 --- /dev/null +++ b/macros/tables/snowflake/pit.yml @@ -0,0 +1,62 @@ +version: 2 + +macros: + - name: snowflake__pit + description: > + Snowflake-specific adapter implementation of the pit macro. Called automatically via adapter.dispatch() + when running on Snowflake. Supports the snapshot_optimization parameter for more efficient incremental + loads on large snapshot tables. See the parent pit macro for full parameter documentation. + arguments: + - name: pit_type + type: string + description: > + String to insert into the 'pit_type' column, prefixed by !. Optional. + - name: tracked_entity + type: string + description: > + Name of the tracked Hub entity. + - name: hashkey + type: string + description: > + Name of the hashkey column inside the tracked Hub. + - name: sat_names + type: list of strings + description: > + List of all satellite names to include in this PIT table. + - name: ldts + type: string + description: > + Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: sdts + type: string + description: > + Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: custom_rsrc + type: string + description: > + Custom string to insert into the rsrc column. Optional. + - name: ledts + type: string + description: > + Name of the load-end-date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: snapshot_relation + type: string + description: > + Name of the snapshot relation model. + - name: snapshot_trigger_column + type: string + description: > + Name of the boolean column that identifies active snapshots. + - name: dimension_key + type: string + description: > + Desired name of the dimension key inside the PIT table. + - name: refer_to_ghost_records + type: boolean + description: > + Whether a NULL satellite hashkey should be replaced by the unknown key. Optional, default True. + - name: snapshot_optimization + type: boolean + description: > + When True in incremental mode, only considers snapshots newer than or equal to the max sdts in + the existing PIT table. Requires a unique_key constraint on the model. Snowflake only. Default False. diff --git a/macros/tables/snowflake/rec_track_sat.yml b/macros/tables/snowflake/rec_track_sat.yml new file mode 100644 index 00000000..634a3a04 --- /dev/null +++ b/macros/tables/snowflake/rec_track_sat.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: snowflake__rec_track_sat + description: > + Snowflake-specific adapter implementation of the rec_track_sat macro. Called automatically via adapter.dispatch() + when running on Snowflake. See the parent rec_track_sat macro for full parameter documentation. + arguments: + - name: tracked_hashkey + type: string + description: > + The name of the hashkey column you want to track. + - name: source_models + type: dictionary + description: > + Dictionary with source model information, including 'rsrc_static' and optionally 'hk_column'. + - name: src_ldts + type: string + description: > + Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: > + Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: src_stg + type: string + description: > + Name of the source stage model. Optional, defaults to 'datavault4dbt.stg_alias'. + - name: disable_hwm + type: boolean + description: > + Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: > + Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/snowflake/ref_hub.yml b/macros/tables/snowflake/ref_hub.yml new file mode 100644 index 00000000..1ae3c275 --- /dev/null +++ b/macros/tables/snowflake/ref_hub.yml @@ -0,0 +1,28 @@ +version: 2 + +macros: + - name: snowflake__ref_hub + description: > + Snowflake-specific adapter implementation of the ref_hub macro. Called automatically via adapter.dispatch() + when running on Snowflake. See the parent ref_hub macro for full parameter documentation. + arguments: + - name: ref_keys + type: string | list of strings + description: > + Name of the reference key(s) available in the source model(s). + - name: src_ldts + type: string + description: > + Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: > + Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: > + Dictionary with information about the source models. + - name: additional_columns + type: list | string + description: > + Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/snowflake/ref_sat_v0.yml b/macros/tables/snowflake/ref_sat_v0.yml new file mode 100644 index 00000000..17885d73 --- /dev/null +++ b/macros/tables/snowflake/ref_sat_v0.yml @@ -0,0 +1,45 @@ +version: 2 + +macros: + - name: snowflake__ref_sat_v0 + description: > + Snowflake-specific adapter implementation of the ref_sat_v0 macro. Called automatically via adapter.dispatch() + when running on Snowflake. Uses QUALIFY for deduplication. See the parent ref_sat_v0 macro for full + parameter documentation. + arguments: + - name: parent_ref_keys + type: string | list of strings + description: > + Name of the reference key(s) of the parent ref_hub. + - name: src_hashdiff + type: string + description: > + Name of the hashdiff column for this reference satellite. + - name: src_payload + type: list of strings + description: > + List of descriptive attribute columns. + - name: src_ldts + type: string + description: > + Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: > + Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: > + Name of the underlying staging model. + - name: disable_hwm + type: boolean + description: > + Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: > + Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: > + Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/snowflake/ref_sat_v1.yml b/macros/tables/snowflake/ref_sat_v1.yml new file mode 100644 index 00000000..2d9b1040 --- /dev/null +++ b/macros/tables/snowflake/ref_sat_v1.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: snowflake__ref_sat_v1 + description: > + Snowflake-specific adapter implementation of the ref_sat_v1 macro. Called automatically via adapter.dispatch() + when running on Snowflake. See the parent ref_sat_v1 macro for full parameter documentation. + arguments: + - name: ref_sat_v0 + type: string + description: > + Name of the underlying ref_sat_v0 dbt model. + - name: ref_keys + type: string | list of strings + description: > + Name(s) of the reference key(s) in the underlying reference sat v0. + - name: hashdiff + type: string + description: > + Name of the Hashdiff column in the underlying reference sat v0. + - name: src_ldts + type: string + description: > + Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: > + Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: > + Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: > + Whether to add an is_current flag. Optional, default False. diff --git a/macros/tables/snowflake/ref_table.yml b/macros/tables/snowflake/ref_table.yml new file mode 100644 index 00000000..1c1730d5 --- /dev/null +++ b/macros/tables/snowflake/ref_table.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: snowflake__ref_table + description: > + Snowflake-specific adapter implementation of the ref_table macro. Called automatically via adapter.dispatch() + when running on Snowflake. See the parent ref_table macro for full parameter documentation. + arguments: + - name: ref_hub + type: string + description: > + Name of the underlying ref_hub model. + - name: src_ldts + type: string + description: > + Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: > + Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ref_satellites + type: string | list of strings + description: > + Name(s) of the reference satellites to include. + - name: historized + type: string + description: > + Possible values are 'full', 'latest', or 'snapshot'. + - name: snapshot_relation + type: string + description: > + Only required if historized is 'snapshot'. Name of the snapshot_v1 model. + - name: snapshot_trigger_column + type: string + description: > + Only required if historized is 'snapshot'. Trigger column name in the snapshot relation. diff --git a/macros/tables/snowflake/sat_v0.yml b/macros/tables/snowflake/sat_v0.yml new file mode 100644 index 00000000..5003da9f --- /dev/null +++ b/macros/tables/snowflake/sat_v0.yml @@ -0,0 +1,45 @@ +version: 2 + +macros: + - name: snowflake__sat_v0 + description: > + Snowflake-specific adapter implementation of the sat_v0 macro. Called automatically via adapter.dispatch() + when running on Snowflake. Uses QUALIFY for intra-batch deduplication and a run_query call for the High + Water Mark. See the parent sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: > + Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: > + Name of the hashdiff column for this satellite. + - name: src_payload + type: list of strings + description: > + List of descriptive attribute columns that form the satellite payload. + - name: source_model + type: string + description: > + Name of the underlying staging model. + - name: src_ldts + type: string + description: > + Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: > + Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: > + Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: > + Whether the source contains only one batch. Skips QUALIFY deduplication. Optional, default False. + - name: additional_columns + type: list | string + description: > + Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/snowflake/sat_v1.yml b/macros/tables/snowflake/sat_v1.yml new file mode 100644 index 00000000..308be0cd --- /dev/null +++ b/macros/tables/snowflake/sat_v1.yml @@ -0,0 +1,40 @@ +version: 2 + +macros: + - name: snowflake__sat_v1 + description: > + Snowflake-specific adapter implementation of the sat_v1 macro. Called automatically via adapter.dispatch() + when running on Snowflake. See the parent sat_v1 macro for full parameter documentation. + arguments: + - name: sat_v0 + type: string + description: > + Name of the underlying version 0 satellite. + - name: hashkey + type: string + description: > + Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: > + Name of the hashdiff column inside the version 0 satellite. + - name: src_ldts + type: string + description: > + Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: > + Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: > + Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: > + Whether to add an is_current flag derived from the ledts column. Optional, default False. + - name: include_payload + type: boolean + description: > + Whether to include the payload columns from sat v0. Optional, default True. diff --git a/macros/tables/sqlserver/control_snap_v0.yml b/macros/tables/sqlserver/control_snap_v0.yml new file mode 100644 index 00000000..05fff615 --- /dev/null +++ b/macros/tables/sqlserver/control_snap_v0.yml @@ -0,0 +1,21 @@ +version: 2 + +macros: + - name: sqlserver__control_snap_v0 + description: > + SQL Server-specific adapter implementation of the control_snap_v0 macro. Called automatically via + adapter.dispatch() when running on SQL Server. See the parent control_snap_v0 macro for full parameter + documentation. + arguments: + - name: start_date + type: timestamp + description: Earliest timestamp for the snapshot table. Time part must be '00:00:00'. + - name: daily_snapshot_time + type: time + description: Time that daily snapshots should have (e.g. '07:00:00'). + - name: sdts_alias + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: end_date + type: timestamp + description: Latest timestamp for the snapshot table. Optional. diff --git a/macros/tables/sqlserver/control_snap_v1.yml b/macros/tables/sqlserver/control_snap_v1.yml new file mode 100644 index 00000000..e66ac477 --- /dev/null +++ b/macros/tables/sqlserver/control_snap_v1.yml @@ -0,0 +1,20 @@ +version: 2 + +macros: + - name: sqlserver__control_snap_v1 + description: > + SQL Server-specific adapter implementation of the control_snap_v1 macro. Called automatically via + adapter.dispatch() when running on SQL Server. See the parent control_snap_v1 macro for full parameter + documentation. + arguments: + - name: control_snap_v0 + type: string + description: The name of the underlying version 0 control snapshot table. + - name: log_logic + type: dictionary + description: > + Defining the desired durations of each granularity ('daily', 'weekly', 'monthly', 'yearly'). + Optional — if not set, all snapshots remain active. + - name: sdts_alias + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. diff --git a/macros/tables/sqlserver/eff_sat_v0.yml b/macros/tables/sqlserver/eff_sat_v0.yml new file mode 100644 index 00000000..d1f8f11d --- /dev/null +++ b/macros/tables/sqlserver/eff_sat_v0.yml @@ -0,0 +1,33 @@ +version: 2 + +macros: + - name: sqlserver__eff_sat_v0 + description: > + SQL Server-specific adapter implementation of the eff_sat_v0 macro. Called automatically via + adapter.dispatch() when running on SQL Server. Uses a lag_source_data CTE instead of QUALIFY. + See the parent eff_sat_v0 macro for full parameter documentation. + arguments: + - name: tracked_hashkey + type: string + description: Name of the hashkey column to be tracked. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: is_active_alias + type: string + description: Name of the active flag column. Optional, defaults to 'datavault4dbt.is_active_alias'. + - name: source_model + type: string + description: Name of the source model. + - name: source_is_single_batch + type: boolean + description: REQUIRED. Whether the source contains only one batch. Must be explicitly set. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be disabled. Optional. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/sqlserver/hub.yml b/macros/tables/sqlserver/hub.yml new file mode 100644 index 00000000..5bc183bc --- /dev/null +++ b/macros/tables/sqlserver/hub.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: sqlserver__hub + description: > + SQL Server-specific adapter implementation of the hub macro. Called automatically via adapter.dispatch() + when running on SQL Server. Uses a lag_source_data CTE approach instead of QUALIFY for deduplication. + See the parent hub macro for full parameter documentation. + arguments: + - name: hashkey + type: string + description: Name of the hashkey column inside the stage, used as PK of the Hub. + - name: business_keys + type: string | list of strings + description: Name(s) of the business key columns loaded into the hub. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include in the Hub. Optional, defaults to empty list. diff --git a/macros/tables/sqlserver/link.yml b/macros/tables/sqlserver/link.yml new file mode 100644 index 00000000..148a0824 --- /dev/null +++ b/macros/tables/sqlserver/link.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: sqlserver__link + description: > + SQL Server-specific adapter implementation of the link macro. Called automatically via adapter.dispatch() + when running on SQL Server. Uses a lag_source_data CTE approach instead of QUALIFY for deduplication. + See the parent link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: Name of the link hashkey column inside the stage. + - name: foreign_hashkeys + type: list of strings + description: List of all hashkey columns inside the link that refer to other hub entities. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include in the Link. Optional, defaults to empty list. diff --git a/macros/tables/sqlserver/ma_sat_v0.yml b/macros/tables/sqlserver/ma_sat_v0.yml new file mode 100644 index 00000000..6c9d8c51 --- /dev/null +++ b/macros/tables/sqlserver/ma_sat_v0.yml @@ -0,0 +1,39 @@ +version: 2 + +macros: + - name: sqlserver__ma_sat_v0 + description: > + SQL Server-specific adapter implementation of the ma_sat_v0 macro. Called automatically via + adapter.dispatch() when running on SQL Server. Uses a lag_source_data CTE instead of QUALIFY for + intra-batch deduplication. See the parent ma_sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this satellite. + - name: src_ma_key + type: string | list of strings + description: Name(s) of the multi-active key column(s). + - name: src_payload + type: list of strings + description: List of descriptive attribute columns (excluding the multi-active key). + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Skips lag CTE deduplication. Optional, default False. diff --git a/macros/tables/sqlserver/ma_sat_v1.yml b/macros/tables/sqlserver/ma_sat_v1.yml new file mode 100644 index 00000000..b40f3b3d --- /dev/null +++ b/macros/tables/sqlserver/ma_sat_v1.yml @@ -0,0 +1,33 @@ +version: 2 + +macros: + - name: sqlserver__ma_sat_v1 + description: > + SQL Server-specific adapter implementation of the ma_sat_v1 macro. Called automatically via + adapter.dispatch() when running on SQL Server. See the parent ma_sat_v1 macro for full parameter + documentation. + arguments: + - name: sat_v0 + type: string + description: Name of the underlying version 0 multi-active satellite. + - name: hashkey + type: string + description: Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: Name of the hashdiff column inside the version 0 satellite. + - name: ma_attribute + type: string | list of strings + description: Name of the multi-active attribute(s) in the v0 satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag derived from the ledts column. Optional, default False. diff --git a/macros/tables/sqlserver/nh_link.yml b/macros/tables/sqlserver/nh_link.yml new file mode 100644 index 00000000..9979e30d --- /dev/null +++ b/macros/tables/sqlserver/nh_link.yml @@ -0,0 +1,38 @@ +version: 2 + +macros: + - name: sqlserver__nh_link + description: > + SQL Server-specific adapter implementation of the nh_link macro. Called automatically via adapter.dispatch() + when running on SQL Server. See the parent nh_link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: Name of the non-historized link hashkey column inside the stage. + - name: payload + type: list of strings + description: List of descriptive attribute columns forming the payload. + - name: foreign_hashkeys + type: list of strings + description: List of all hashkey columns inside the link that refer to other hub entities. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: union_strategy + type: "'all' | 'distinct'" + description: Defines how multiple sources should be unioned. Default 'all'. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/sqlserver/nh_sat.yml b/macros/tables/sqlserver/nh_sat.yml new file mode 100644 index 00000000..23290aef --- /dev/null +++ b/macros/tables/sqlserver/nh_sat.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: sqlserver__nh_sat + description: > + SQL Server-specific adapter implementation of the nh_sat macro. Called automatically via adapter.dispatch() + when running on SQL Server. See the parent nh_sat macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns for this non-historized satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/sqlserver/pit.yml b/macros/tables/sqlserver/pit.yml new file mode 100644 index 00000000..8014f30b --- /dev/null +++ b/macros/tables/sqlserver/pit.yml @@ -0,0 +1,47 @@ +version: 2 + +macros: + - name: sqlserver__pit + description: > + SQL Server-specific adapter implementation of the pit macro. Called automatically via adapter.dispatch() + when running on SQL Server. See the parent pit macro for full parameter documentation. + arguments: + - name: pit_type + type: string + description: String to insert into the 'pit_type' column, prefixed by !. Optional. + - name: tracked_entity + type: string + description: Name of the tracked Hub entity. + - name: hashkey + type: string + description: Name of the hashkey column inside the tracked Hub. + - name: sat_names + type: list of strings + description: List of all satellite names to include in this PIT table. + - name: ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: sdts + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: custom_rsrc + type: string + description: Custom string to insert into the rsrc column. Optional. + - name: ledts + type: string + description: Name of the load-end-date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: snapshot_relation + type: string + description: Name of the snapshot relation model. + - name: snapshot_trigger_column + type: string + description: Name of the boolean column that identifies active snapshots. + - name: dimension_key + type: string + description: Desired name of the dimension key inside the PIT table. + - name: refer_to_ghost_records + type: boolean + description: Whether a NULL satellite hashkey should be replaced by the unknown key. Optional, default True. + - name: snapshot_optimization + type: boolean + description: Whether to apply snapshot optimization logic. Optional, default False. diff --git a/macros/tables/sqlserver/rec_track_sat.yml b/macros/tables/sqlserver/rec_track_sat.yml new file mode 100644 index 00000000..2d8c4de5 --- /dev/null +++ b/macros/tables/sqlserver/rec_track_sat.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: sqlserver__rec_track_sat + description: > + SQL Server-specific adapter implementation of the rec_track_sat macro. Called automatically via + adapter.dispatch() when running on SQL Server. See the parent rec_track_sat macro for full parameter + documentation. + arguments: + - name: tracked_hashkey + type: string + description: The name of the hashkey column you want to track. + - name: source_models + type: dictionary + description: Dictionary with source model information, including 'rsrc_static' and optionally 'hk_column'. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: src_stg + type: string + description: Name of the source stage model. Optional, defaults to 'datavault4dbt.stg_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/sqlserver/ref_hub.yml b/macros/tables/sqlserver/ref_hub.yml new file mode 100644 index 00000000..2306a4b7 --- /dev/null +++ b/macros/tables/sqlserver/ref_hub.yml @@ -0,0 +1,23 @@ +version: 2 + +macros: + - name: sqlserver__ref_hub + description: > + SQL Server-specific adapter implementation of the ref_hub macro. Called automatically via adapter.dispatch() + when running on SQL Server. See the parent ref_hub macro for full parameter documentation. + arguments: + - name: ref_keys + type: string | list of strings + description: Name of the reference key(s) available in the source model(s). + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/sqlserver/ref_sat_v0.yml b/macros/tables/sqlserver/ref_sat_v0.yml new file mode 100644 index 00000000..0a6d68b2 --- /dev/null +++ b/macros/tables/sqlserver/ref_sat_v0.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: sqlserver__ref_sat_v0 + description: > + SQL Server-specific adapter implementation of the ref_sat_v0 macro. Called automatically via + adapter.dispatch() when running on SQL Server. Uses a lag_source_data CTE instead of QUALIFY for + deduplication. See the parent ref_sat_v0 macro for full parameter documentation. + arguments: + - name: parent_ref_keys + type: string | list of strings + description: Name of the reference key(s) of the parent ref_hub. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this reference satellite. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/sqlserver/ref_sat_v1.yml b/macros/tables/sqlserver/ref_sat_v1.yml new file mode 100644 index 00000000..3cd9ad6a --- /dev/null +++ b/macros/tables/sqlserver/ref_sat_v1.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: sqlserver__ref_sat_v1 + description: > + SQL Server-specific adapter implementation of the ref_sat_v1 macro. Called automatically via + adapter.dispatch() when running on SQL Server. See the parent ref_sat_v1 macro for full parameter + documentation. + arguments: + - name: ref_sat_v0 + type: string + description: Name of the underlying ref_sat_v0 dbt model. + - name: ref_keys + type: string | list of strings + description: Name(s) of the reference key(s) in the underlying reference sat v0. + - name: hashdiff + type: string + description: Name of the Hashdiff column in the underlying reference sat v0. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag. Optional, default False. diff --git a/macros/tables/sqlserver/ref_table.yml b/macros/tables/sqlserver/ref_table.yml new file mode 100644 index 00000000..ee760cf6 --- /dev/null +++ b/macros/tables/sqlserver/ref_table.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: sqlserver__ref_table + description: > + SQL Server-specific adapter implementation of the ref_table macro. Called automatically via adapter.dispatch() + when running on SQL Server. See the parent ref_table macro for full parameter documentation. + arguments: + - name: ref_hub + type: string + description: Name of the underlying ref_hub model. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ref_satellites + type: string | list of strings + description: Name(s) of the reference satellites to include. + - name: historized + type: string + description: Possible values are 'full', 'latest', or 'snapshot'. + - name: snapshot_relation + type: string + description: Only required if historized is 'snapshot'. Name of the snapshot_v1 model. + - name: snapshot_trigger_column + type: string + description: Only required if historized is 'snapshot'. Trigger column name in the snapshot relation. diff --git a/macros/tables/sqlserver/sat_v0.yml b/macros/tables/sqlserver/sat_v0.yml new file mode 100644 index 00000000..cd014913 --- /dev/null +++ b/macros/tables/sqlserver/sat_v0.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: sqlserver__sat_v0 + description: > + SQL Server-specific adapter implementation of the sat_v0 macro. Called automatically via adapter.dispatch() + when running on SQL Server. Uses a lag_source_data CTE instead of QUALIFY for intra-batch deduplication. + See the parent sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this satellite. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns that form the satellite payload. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Skips lag CTE deduplication. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/sqlserver/sat_v1.yml b/macros/tables/sqlserver/sat_v1.yml new file mode 100644 index 00000000..18244a36 --- /dev/null +++ b/macros/tables/sqlserver/sat_v1.yml @@ -0,0 +1,32 @@ +version: 2 + +macros: + - name: sqlserver__sat_v1 + description: > + SQL Server-specific adapter implementation of the sat_v1 macro. Called automatically via adapter.dispatch() + when running on SQL Server. See the parent sat_v1 macro for full parameter documentation. + arguments: + - name: sat_v0 + type: string + description: Name of the underlying version 0 satellite. + - name: hashkey + type: string + description: Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: Name of the hashdiff column inside the version 0 satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag derived from the ledts column. Optional, default False. + - name: include_payload + type: boolean + description: Whether to include the payload columns from sat v0. Optional, default True. diff --git a/macros/tables/synapse/control_snap_v0.yml b/macros/tables/synapse/control_snap_v0.yml new file mode 100644 index 00000000..0d67449a --- /dev/null +++ b/macros/tables/synapse/control_snap_v0.yml @@ -0,0 +1,21 @@ +version: 2 + +macros: + - name: synapse__control_snap_v0 + description: > + Azure Synapse-specific adapter implementation of the control_snap_v0 macro. Called automatically via + adapter.dispatch() when running on Synapse. See the parent control_snap_v0 macro for full parameter + documentation. + arguments: + - name: start_date + type: timestamp + description: Earliest timestamp for the snapshot table. Time part must be '00:00:00'. + - name: daily_snapshot_time + type: time + description: Time that daily snapshots should have (e.g. '07:00:00'). + - name: sdts_alias + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: end_date + type: timestamp + description: Latest timestamp for the snapshot table. Optional. diff --git a/macros/tables/synapse/control_snap_v1.yml b/macros/tables/synapse/control_snap_v1.yml new file mode 100644 index 00000000..5abf2f68 --- /dev/null +++ b/macros/tables/synapse/control_snap_v1.yml @@ -0,0 +1,20 @@ +version: 2 + +macros: + - name: synapse__control_snap_v1 + description: > + Azure Synapse-specific adapter implementation of the control_snap_v1 macro. Called automatically via + adapter.dispatch() when running on Synapse. See the parent control_snap_v1 macro for full parameter + documentation. + arguments: + - name: control_snap_v0 + type: string + description: The name of the underlying version 0 control snapshot table. + - name: log_logic + type: dictionary + description: > + Defining the desired durations of each granularity ('daily', 'weekly', 'monthly', 'yearly'). + Optional — if not set, all snapshots remain active. + - name: sdts_alias + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. diff --git a/macros/tables/synapse/eff_sat_v0.yml b/macros/tables/synapse/eff_sat_v0.yml new file mode 100644 index 00000000..5f6b9124 --- /dev/null +++ b/macros/tables/synapse/eff_sat_v0.yml @@ -0,0 +1,33 @@ +version: 2 + +macros: + - name: synapse__eff_sat_v0 + description: > + Azure Synapse-specific adapter implementation of the eff_sat_v0 macro. Called automatically via + adapter.dispatch() when running on Synapse. Uses a lag_source_data CTE instead of QUALIFY. + See the parent eff_sat_v0 macro for full parameter documentation. + arguments: + - name: tracked_hashkey + type: string + description: Name of the hashkey column to be tracked. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: is_active_alias + type: string + description: Name of the active flag column. Optional, defaults to 'datavault4dbt.is_active_alias'. + - name: source_model + type: string + description: Name of the source model. + - name: source_is_single_batch + type: boolean + description: REQUIRED. Whether the source contains only one batch. Must be explicitly set. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be disabled. Optional. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/synapse/hub.yml b/macros/tables/synapse/hub.yml new file mode 100644 index 00000000..d247c3fc --- /dev/null +++ b/macros/tables/synapse/hub.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: synapse__hub + description: > + Azure Synapse-specific adapter implementation of the hub macro. Called automatically via adapter.dispatch() + when running on Synapse. Uses a lag_source_data CTE approach instead of QUALIFY for deduplication. + See the parent hub macro for full parameter documentation. + arguments: + - name: hashkey + type: string + description: Name of the hashkey column inside the stage, used as PK of the Hub. + - name: business_keys + type: string | list of strings + description: Name(s) of the business key columns loaded into the hub. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include in the Hub. Optional, defaults to empty list. diff --git a/macros/tables/synapse/link.yml b/macros/tables/synapse/link.yml new file mode 100644 index 00000000..011244a8 --- /dev/null +++ b/macros/tables/synapse/link.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: synapse__link + description: > + Azure Synapse-specific adapter implementation of the link macro. Called automatically via adapter.dispatch() + when running on Synapse. Uses a lag_source_data CTE approach instead of QUALIFY for deduplication. + See the parent link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: Name of the link hashkey column inside the stage. + - name: foreign_hashkeys + type: list of strings + description: List of all hashkey columns inside the link that refer to other hub entities. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include in the Link. Optional, defaults to empty list. diff --git a/macros/tables/synapse/ma_sat_v0.yml b/macros/tables/synapse/ma_sat_v0.yml new file mode 100644 index 00000000..021f8ca0 --- /dev/null +++ b/macros/tables/synapse/ma_sat_v0.yml @@ -0,0 +1,39 @@ +version: 2 + +macros: + - name: synapse__ma_sat_v0 + description: > + Azure Synapse-specific adapter implementation of the ma_sat_v0 macro. Called automatically via + adapter.dispatch() when running on Synapse. Uses a lag_source_data CTE instead of QUALIFY for + intra-batch deduplication. See the parent ma_sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this satellite. + - name: src_ma_key + type: string | list of strings + description: Name(s) of the multi-active key column(s). + - name: src_payload + type: list of strings + description: List of descriptive attribute columns (excluding the multi-active key). + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Skips lag CTE deduplication. Optional, default False. diff --git a/macros/tables/synapse/ma_sat_v1.yml b/macros/tables/synapse/ma_sat_v1.yml new file mode 100644 index 00000000..1a34c76a --- /dev/null +++ b/macros/tables/synapse/ma_sat_v1.yml @@ -0,0 +1,33 @@ +version: 2 + +macros: + - name: synapse__ma_sat_v1 + description: > + Azure Synapse-specific adapter implementation of the ma_sat_v1 macro. Called automatically via + adapter.dispatch() when running on Synapse. See the parent ma_sat_v1 macro for full parameter + documentation. + arguments: + - name: sat_v0 + type: string + description: Name of the underlying version 0 multi-active satellite. + - name: hashkey + type: string + description: Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: Name of the hashdiff column inside the version 0 satellite. + - name: ma_attribute + type: string | list of strings + description: Name of the multi-active attribute(s) in the v0 satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag derived from the ledts column. Optional, default False. diff --git a/macros/tables/synapse/nh_link.yml b/macros/tables/synapse/nh_link.yml new file mode 100644 index 00000000..a5d6ce14 --- /dev/null +++ b/macros/tables/synapse/nh_link.yml @@ -0,0 +1,38 @@ +version: 2 + +macros: + - name: synapse__nh_link + description: > + Azure Synapse-specific adapter implementation of the nh_link macro. Called automatically via adapter.dispatch() + when running on Synapse. See the parent nh_link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: Name of the non-historized link hashkey column inside the stage. + - name: payload + type: list of strings + description: List of descriptive attribute columns forming the payload. + - name: foreign_hashkeys + type: list of strings + description: List of all hashkey columns inside the link that refer to other hub entities. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: union_strategy + type: "'all' | 'distinct'" + description: Defines how multiple sources should be unioned. Default 'all'. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/synapse/nh_sat.yml b/macros/tables/synapse/nh_sat.yml new file mode 100644 index 00000000..0061f9c5 --- /dev/null +++ b/macros/tables/synapse/nh_sat.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: synapse__nh_sat + description: > + Azure Synapse-specific adapter implementation of the nh_sat macro. Called automatically via adapter.dispatch() + when running on Synapse. See the parent nh_sat macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns for this non-historized satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/synapse/pit.yml b/macros/tables/synapse/pit.yml new file mode 100644 index 00000000..07a76721 --- /dev/null +++ b/macros/tables/synapse/pit.yml @@ -0,0 +1,47 @@ +version: 2 + +macros: + - name: synapse__pit + description: > + Azure Synapse-specific adapter implementation of the pit macro. Called automatically via adapter.dispatch() + when running on Synapse. See the parent pit macro for full parameter documentation. + arguments: + - name: pit_type + type: string + description: String to insert into the 'pit_type' column, prefixed by !. Optional. + - name: tracked_entity + type: string + description: Name of the tracked Hub entity. + - name: hashkey + type: string + description: Name of the hashkey column inside the tracked Hub. + - name: sat_names + type: list of strings + description: List of all satellite names to include in this PIT table. + - name: ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: sdts + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: custom_rsrc + type: string + description: Custom string to insert into the rsrc column. Optional. + - name: ledts + type: string + description: Name of the load-end-date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: snapshot_relation + type: string + description: Name of the snapshot relation model. + - name: snapshot_trigger_column + type: string + description: Name of the boolean column that identifies active snapshots. + - name: dimension_key + type: string + description: Desired name of the dimension key inside the PIT table. + - name: refer_to_ghost_records + type: boolean + description: Whether a NULL satellite hashkey should be replaced by the unknown key. Optional, default True. + - name: snapshot_optimization + type: boolean + description: Whether to apply snapshot optimization logic. Optional, default False. diff --git a/macros/tables/synapse/rec_track_sat.yml b/macros/tables/synapse/rec_track_sat.yml new file mode 100644 index 00000000..5a1d82a7 --- /dev/null +++ b/macros/tables/synapse/rec_track_sat.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: synapse__rec_track_sat + description: > + Azure Synapse-specific adapter implementation of the rec_track_sat macro. Called automatically via + adapter.dispatch() when running on Synapse. See the parent rec_track_sat macro for full parameter + documentation. + arguments: + - name: tracked_hashkey + type: string + description: The name of the hashkey column you want to track. + - name: source_models + type: dictionary + description: Dictionary with source model information, including 'rsrc_static' and optionally 'hk_column'. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: src_stg + type: string + description: Name of the source stage model. Optional, defaults to 'datavault4dbt.stg_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/synapse/ref_hub.yml b/macros/tables/synapse/ref_hub.yml new file mode 100644 index 00000000..0a6960bc --- /dev/null +++ b/macros/tables/synapse/ref_hub.yml @@ -0,0 +1,23 @@ +version: 2 + +macros: + - name: synapse__ref_hub + description: > + Azure Synapse-specific adapter implementation of the ref_hub macro. Called automatically via adapter.dispatch() + when running on Synapse. See the parent ref_hub macro for full parameter documentation. + arguments: + - name: ref_keys + type: string | list of strings + description: Name of the reference key(s) available in the source model(s). + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/synapse/ref_sat_v0.yml b/macros/tables/synapse/ref_sat_v0.yml new file mode 100644 index 00000000..4905e11b --- /dev/null +++ b/macros/tables/synapse/ref_sat_v0.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: synapse__ref_sat_v0 + description: > + Azure Synapse-specific adapter implementation of the ref_sat_v0 macro. Called automatically via + adapter.dispatch() when running on Synapse. Uses a lag_source_data CTE instead of QUALIFY for + deduplication. See the parent ref_sat_v0 macro for full parameter documentation. + arguments: + - name: parent_ref_keys + type: string | list of strings + description: Name of the reference key(s) of the parent ref_hub. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this reference satellite. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/synapse/ref_sat_v1.yml b/macros/tables/synapse/ref_sat_v1.yml new file mode 100644 index 00000000..d9c81d3e --- /dev/null +++ b/macros/tables/synapse/ref_sat_v1.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: synapse__ref_sat_v1 + description: > + Azure Synapse-specific adapter implementation of the ref_sat_v1 macro. Called automatically via + adapter.dispatch() when running on Synapse. See the parent ref_sat_v1 macro for full parameter + documentation. + arguments: + - name: ref_sat_v0 + type: string + description: Name of the underlying ref_sat_v0 dbt model. + - name: ref_keys + type: string | list of strings + description: Name(s) of the reference key(s) in the underlying reference sat v0. + - name: hashdiff + type: string + description: Name of the Hashdiff column in the underlying reference sat v0. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag. Optional, default False. diff --git a/macros/tables/synapse/ref_table.yml b/macros/tables/synapse/ref_table.yml new file mode 100644 index 00000000..1452966f --- /dev/null +++ b/macros/tables/synapse/ref_table.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: synapse__ref_table + description: > + Azure Synapse-specific adapter implementation of the ref_table macro. Called automatically via + adapter.dispatch() when running on Synapse. See the parent ref_table macro for full parameter + documentation. + arguments: + - name: ref_hub + type: string + description: Name of the underlying ref_hub model. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ref_satellites + type: string | list of strings + description: Name(s) of the reference satellites to include. + - name: historized + type: string + description: Possible values are 'full', 'latest', or 'snapshot'. + - name: snapshot_relation + type: string + description: Only required if historized is 'snapshot'. Name of the snapshot_v1 model. + - name: snapshot_trigger_column + type: string + description: Only required if historized is 'snapshot'. Trigger column name in the snapshot relation. diff --git a/macros/tables/synapse/sat_v0.yml b/macros/tables/synapse/sat_v0.yml new file mode 100644 index 00000000..fbb385b8 --- /dev/null +++ b/macros/tables/synapse/sat_v0.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: synapse__sat_v0 + description: > + Azure Synapse-specific adapter implementation of the sat_v0 macro. Called automatically via adapter.dispatch() + when running on Synapse. Uses a lag_source_data CTE instead of QUALIFY for intra-batch deduplication. + See the parent sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this satellite. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns that form the satellite payload. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Skips lag CTE deduplication. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/synapse/sat_v1.yml b/macros/tables/synapse/sat_v1.yml new file mode 100644 index 00000000..072d6e32 --- /dev/null +++ b/macros/tables/synapse/sat_v1.yml @@ -0,0 +1,32 @@ +version: 2 + +macros: + - name: synapse__sat_v1 + description: > + Azure Synapse-specific adapter implementation of the sat_v1 macro. Called automatically via adapter.dispatch() + when running on Synapse. See the parent sat_v1 macro for full parameter documentation. + arguments: + - name: sat_v0 + type: string + description: Name of the underlying version 0 satellite. + - name: hashkey + type: string + description: Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: Name of the hashdiff column inside the version 0 satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag derived from the ledts column. Optional, default False. + - name: include_payload + type: boolean + description: Whether to include the payload columns from sat v0. Optional, default True. diff --git a/macros/tables/trino/control_snap_v0.yml b/macros/tables/trino/control_snap_v0.yml new file mode 100644 index 00000000..7f75f43d --- /dev/null +++ b/macros/tables/trino/control_snap_v0.yml @@ -0,0 +1,21 @@ +version: 2 + +macros: + - name: trino__control_snap_v0 + description: > + Trino-specific adapter implementation of the control_snap_v0 macro. Called automatically via + adapter.dispatch() when running on Trino. See the parent control_snap_v0 macro for full parameter + documentation. + arguments: + - name: start_date + type: timestamp + description: Earliest timestamp for the snapshot table. Time part must be '00:00:00'. + - name: daily_snapshot_time + type: time + description: Time that daily snapshots should have (e.g. '07:00:00'). + - name: sdts_alias + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: end_date + type: timestamp + description: Latest timestamp for the snapshot table. Optional. diff --git a/macros/tables/trino/control_snap_v1.yml b/macros/tables/trino/control_snap_v1.yml new file mode 100644 index 00000000..18207abc --- /dev/null +++ b/macros/tables/trino/control_snap_v1.yml @@ -0,0 +1,20 @@ +version: 2 + +macros: + - name: trino__control_snap_v1 + description: > + Trino-specific adapter implementation of the control_snap_v1 macro. Called automatically via + adapter.dispatch() when running on Trino. See the parent control_snap_v1 macro for full parameter + documentation. + arguments: + - name: control_snap_v0 + type: string + description: The name of the underlying version 0 control snapshot table. + - name: log_logic + type: dictionary + description: > + Defining the desired durations of each granularity ('daily', 'weekly', 'monthly', 'yearly'). + Optional — if not set, all snapshots remain active. + - name: sdts_alias + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. diff --git a/macros/tables/trino/eff_sat_v0.yml b/macros/tables/trino/eff_sat_v0.yml new file mode 100644 index 00000000..bab55df2 --- /dev/null +++ b/macros/tables/trino/eff_sat_v0.yml @@ -0,0 +1,33 @@ +version: 2 + +macros: + - name: trino__eff_sat_v0 + description: > + Trino-specific adapter implementation of the eff_sat_v0 macro. Called automatically via adapter.dispatch() + when running on Trino. Uses a lag_source_data CTE instead of QUALIFY. + See the parent eff_sat_v0 macro for full parameter documentation. + arguments: + - name: tracked_hashkey + type: string + description: Name of the hashkey column to be tracked. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: is_active_alias + type: string + description: Name of the active flag column. Optional, defaults to 'datavault4dbt.is_active_alias'. + - name: source_model + type: string + description: Name of the source model. + - name: source_is_single_batch + type: boolean + description: REQUIRED. Whether the source contains only one batch. Must be explicitly set. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be disabled. Optional. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/trino/hub.yml b/macros/tables/trino/hub.yml new file mode 100644 index 00000000..718e1135 --- /dev/null +++ b/macros/tables/trino/hub.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: trino__hub + description: > + Trino-specific adapter implementation of the hub macro. Called automatically via adapter.dispatch() + when running on Trino. Uses a lag_source_data CTE approach instead of QUALIFY for deduplication. + See the parent hub macro for full parameter documentation. + arguments: + - name: hashkey + type: string + description: Name of the hashkey column inside the stage, used as PK of the Hub. + - name: business_keys + type: string | list of strings + description: Name(s) of the business key columns loaded into the hub. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include in the Hub. Optional, defaults to empty list. diff --git a/macros/tables/trino/link.yml b/macros/tables/trino/link.yml new file mode 100644 index 00000000..f86dc54b --- /dev/null +++ b/macros/tables/trino/link.yml @@ -0,0 +1,30 @@ +version: 2 + +macros: + - name: trino__link + description: > + Trino-specific adapter implementation of the link macro. Called automatically via adapter.dispatch() + when running on Trino. Uses a lag_source_data CTE approach instead of QUALIFY for deduplication. + See the parent link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: Name of the link hashkey column inside the stage. + - name: foreign_hashkeys + type: list of strings + description: List of all hashkey columns inside the link that refer to other hub entities. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include in the Link. Optional, defaults to empty list. diff --git a/macros/tables/trino/ma_sat_v0.yml b/macros/tables/trino/ma_sat_v0.yml new file mode 100644 index 00000000..00789967 --- /dev/null +++ b/macros/tables/trino/ma_sat_v0.yml @@ -0,0 +1,39 @@ +version: 2 + +macros: + - name: trino__ma_sat_v0 + description: > + Trino-specific adapter implementation of the ma_sat_v0 macro. Called automatically via adapter.dispatch() + when running on Trino. Uses a lag_source_data CTE instead of QUALIFY for intra-batch deduplication. + See the parent ma_sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this satellite. + - name: src_ma_key + type: string | list of strings + description: Name(s) of the multi-active key column(s). + - name: src_payload + type: list of strings + description: List of descriptive attribute columns (excluding the multi-active key). + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Skips lag CTE deduplication. Optional, default False. diff --git a/macros/tables/trino/ma_sat_v1.yml b/macros/tables/trino/ma_sat_v1.yml new file mode 100644 index 00000000..1990141f --- /dev/null +++ b/macros/tables/trino/ma_sat_v1.yml @@ -0,0 +1,32 @@ +version: 2 + +macros: + - name: trino__ma_sat_v1 + description: > + Trino-specific adapter implementation of the ma_sat_v1 macro. Called automatically via adapter.dispatch() + when running on Trino. See the parent ma_sat_v1 macro for full parameter documentation. + arguments: + - name: sat_v0 + type: string + description: Name of the underlying version 0 multi-active satellite. + - name: hashkey + type: string + description: Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: Name of the hashdiff column inside the version 0 satellite. + - name: ma_attribute + type: string | list of strings + description: Name of the multi-active attribute(s) in the v0 satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag derived from the ledts column. Optional, default False. diff --git a/macros/tables/trino/nh_link.yml b/macros/tables/trino/nh_link.yml new file mode 100644 index 00000000..8cae7207 --- /dev/null +++ b/macros/tables/trino/nh_link.yml @@ -0,0 +1,38 @@ +version: 2 + +macros: + - name: trino__nh_link + description: > + Trino-specific adapter implementation of the nh_link macro. Called automatically via adapter.dispatch() + when running on Trino. See the parent nh_link macro for full parameter documentation. + arguments: + - name: link_hashkey + type: string + description: Name of the non-historized link hashkey column inside the stage. + - name: payload + type: list of strings + description: List of descriptive attribute columns forming the payload. + - name: foreign_hashkeys + type: list of strings + description: List of all hashkey columns inside the link that refer to other hub entities. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: union_strategy + type: "'all' | 'distinct'" + description: Defines how multiple sources should be unioned. Default 'all'. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/trino/nh_sat.yml b/macros/tables/trino/nh_sat.yml new file mode 100644 index 00000000..a34c3c52 --- /dev/null +++ b/macros/tables/trino/nh_sat.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: trino__nh_sat + description: > + Trino-specific adapter implementation of the nh_sat macro. Called automatically via adapter.dispatch() + when running on Trino. See the parent nh_sat macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns for this non-historized satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/trino/pit.yml b/macros/tables/trino/pit.yml new file mode 100644 index 00000000..e175a84d --- /dev/null +++ b/macros/tables/trino/pit.yml @@ -0,0 +1,47 @@ +version: 2 + +macros: + - name: trino__pit + description: > + Trino-specific adapter implementation of the pit macro. Called automatically via adapter.dispatch() + when running on Trino. See the parent pit macro for full parameter documentation. + arguments: + - name: pit_type + type: string + description: String to insert into the 'pit_type' column, prefixed by !. Optional. + - name: tracked_entity + type: string + description: Name of the tracked Hub entity. + - name: hashkey + type: string + description: Name of the hashkey column inside the tracked Hub. + - name: sat_names + type: list of strings + description: List of all satellite names to include in this PIT table. + - name: ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: sdts + type: string + description: Name of the snapshot date timestamp column. Optional, defaults to 'datavault4dbt.sdts_alias'. + - name: custom_rsrc + type: string + description: Custom string to insert into the rsrc column. Optional. + - name: ledts + type: string + description: Name of the load-end-date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: snapshot_relation + type: string + description: Name of the snapshot relation model. + - name: snapshot_trigger_column + type: string + description: Name of the boolean column that identifies active snapshots. + - name: dimension_key + type: string + description: Desired name of the dimension key inside the PIT table. + - name: refer_to_ghost_records + type: boolean + description: Whether a NULL satellite hashkey should be replaced by the unknown key. Optional, default True. + - name: snapshot_optimization + type: boolean + description: Whether to apply snapshot optimization logic. Optional, default False. diff --git a/macros/tables/trino/rec_track_sat.yml b/macros/tables/trino/rec_track_sat.yml new file mode 100644 index 00000000..7851562b --- /dev/null +++ b/macros/tables/trino/rec_track_sat.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: trino__rec_track_sat + description: > + Trino-specific adapter implementation of the rec_track_sat macro. Called automatically via adapter.dispatch() + when running on Trino. See the parent rec_track_sat macro for full parameter documentation. + arguments: + - name: tracked_hashkey + type: string + description: The name of the hashkey column you want to track. + - name: source_models + type: dictionary + description: Dictionary with source model information, including 'rsrc_static' and optionally 'hk_column'. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: src_stg + type: string + description: Name of the source stage model. Optional, defaults to 'datavault4dbt.stg_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/trino/ref_hub.yml b/macros/tables/trino/ref_hub.yml new file mode 100644 index 00000000..4207ba15 --- /dev/null +++ b/macros/tables/trino/ref_hub.yml @@ -0,0 +1,23 @@ +version: 2 + +macros: + - name: trino__ref_hub + description: > + Trino-specific adapter implementation of the ref_hub macro. Called automatically via adapter.dispatch() + when running on Trino. See the parent ref_hub macro for full parameter documentation. + arguments: + - name: ref_keys + type: string | list of strings + description: Name of the reference key(s) available in the source model(s). + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_models + type: dictionary + description: Dictionary with information about the source models. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/trino/ref_sat_v0.yml b/macros/tables/trino/ref_sat_v0.yml new file mode 100644 index 00000000..8bc81fb0 --- /dev/null +++ b/macros/tables/trino/ref_sat_v0.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: trino__ref_sat_v0 + description: > + Trino-specific adapter implementation of the ref_sat_v0 macro. Called automatically via adapter.dispatch() + when running on Trino. Uses a lag_source_data CTE instead of QUALIFY for deduplication. + See the parent ref_sat_v0 macro for full parameter documentation. + arguments: + - name: parent_ref_keys + type: string | list of strings + description: Name of the reference key(s) of the parent ref_hub. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this reference satellite. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/trino/ref_sat_v1.yml b/macros/tables/trino/ref_sat_v1.yml new file mode 100644 index 00000000..ee385cd0 --- /dev/null +++ b/macros/tables/trino/ref_sat_v1.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: trino__ref_sat_v1 + description: > + Trino-specific adapter implementation of the ref_sat_v1 macro. Called automatically via adapter.dispatch() + when running on Trino. See the parent ref_sat_v1 macro for full parameter documentation. + arguments: + - name: ref_sat_v0 + type: string + description: Name of the underlying ref_sat_v0 dbt model. + - name: ref_keys + type: string | list of strings + description: Name(s) of the reference key(s) in the underlying reference sat v0. + - name: hashdiff + type: string + description: Name of the Hashdiff column in the underlying reference sat v0. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag. Optional, default False. diff --git a/macros/tables/trino/ref_table.yml b/macros/tables/trino/ref_table.yml new file mode 100644 index 00000000..2349b33b --- /dev/null +++ b/macros/tables/trino/ref_table.yml @@ -0,0 +1,29 @@ +version: 2 + +macros: + - name: trino__ref_table + description: > + Trino-specific adapter implementation of the ref_table macro. Called automatically via adapter.dispatch() + when running on Trino. See the parent ref_table macro for full parameter documentation. + arguments: + - name: ref_hub + type: string + description: Name of the underlying ref_hub model. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ref_satellites + type: string | list of strings + description: Name(s) of the reference satellites to include. + - name: historized + type: string + description: Possible values are 'full', 'latest', or 'snapshot'. + - name: snapshot_relation + type: string + description: Only required if historized is 'snapshot'. Name of the snapshot_v1 model. + - name: snapshot_trigger_column + type: string + description: Only required if historized is 'snapshot'. Trigger column name in the snapshot relation. diff --git a/macros/tables/trino/sat_v0.yml b/macros/tables/trino/sat_v0.yml new file mode 100644 index 00000000..f1bb95d4 --- /dev/null +++ b/macros/tables/trino/sat_v0.yml @@ -0,0 +1,36 @@ +version: 2 + +macros: + - name: trino__sat_v0 + description: > + Trino-specific adapter implementation of the sat_v0 macro. Called automatically via adapter.dispatch() + when running on Trino. Uses a lag_source_data CTE instead of QUALIFY for intra-batch deduplication. + See the parent sat_v0 macro for full parameter documentation. + arguments: + - name: parent_hashkey + type: string + description: Name of the hashkey column of the parent hub or link. + - name: src_hashdiff + type: string + description: Name of the hashdiff column for this satellite. + - name: src_payload + type: list of strings + description: List of descriptive attribute columns that form the satellite payload. + - name: source_model + type: string + description: Name of the underlying staging model. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: disable_hwm + type: boolean + description: Whether the High Water Mark should be turned off. Optional, default False. + - name: source_is_single_batch + type: boolean + description: Whether the source contains only one batch. Skips lag CTE deduplication. Optional, default False. + - name: additional_columns + type: list | string + description: Additional columns to include. Optional, defaults to empty list. diff --git a/macros/tables/trino/sat_v1.yml b/macros/tables/trino/sat_v1.yml new file mode 100644 index 00000000..f7676860 --- /dev/null +++ b/macros/tables/trino/sat_v1.yml @@ -0,0 +1,32 @@ +version: 2 + +macros: + - name: trino__sat_v1 + description: > + Trino-specific adapter implementation of the sat_v1 macro. Called automatically via adapter.dispatch() + when running on Trino. See the parent sat_v1 macro for full parameter documentation. + arguments: + - name: sat_v0 + type: string + description: Name of the underlying version 0 satellite. + - name: hashkey + type: string + description: Name of the parent hashkey column inside the version 0 satellite. + - name: hashdiff + type: string + description: Name of the hashdiff column inside the version 0 satellite. + - name: src_ldts + type: string + description: Name of the ldts column. Optional, defaults to 'datavault4dbt.ldts_alias'. + - name: src_rsrc + type: string + description: Name of the rsrc column. Optional, defaults to 'datavault4dbt.rsrc_alias'. + - name: ledts_alias + type: string + description: Desired alias for the load end date column. Optional, defaults to 'datavault4dbt.ledts_alias'. + - name: add_is_current_flag + type: boolean + description: Whether to add an is_current flag derived from the ledts column. Optional, default False. + - name: include_payload + type: boolean + description: Whether to include the payload columns from sat v0. Optional, default True.