Skip to content

Commit 6cdf9c3

Browse files
tkiehntkiehn
authored
fix(trino): ghost records and column capitalization in stages (#487)
* fix(trino): handling of ghost record datatypes * fix(trino): Remove column capitalization in stage-macro
1 parent bd87115 commit 6cdf9c3

3 files changed

Lines changed: 46 additions & 8 deletions

File tree

‎macros/internal/metadata_processing/escape_column_names.sql‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,4 +214,16 @@
214214

215215
{%- do return(escaped_column_name) -%}
216216

217+
{%- endmacro -%}
218+
219+
220+
{%- macro trino__escape_column_name(column) -%}
221+
222+
{%- set escape_char_left = var('escape_char_left', '"') -%}
223+
{%- set escape_char_right = var('escape_char_right', '"') -%}
224+
225+
{%- set escaped_column_name = escape_char_left ~ column | replace(escape_char_left, '') | replace(escape_char_right, '') | trim ~ escape_char_right | indent(4) -%}
226+
227+
{%- do return(escaped_column_name) -%}
228+
217229
{%- endmacro -%}

‎macros/staging/trino/stage.sql‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -480,7 +480,7 @@ unknown_values AS (
480480
{%- if columns_without_excluded_columns is defined and columns_without_excluded_columns| length > 0 -%}
481481
{# Generating Ghost Records for all source columns, except the ldts, rsrc & edwSequence column #}
482482
{%- for column in columns_without_excluded_columns %}
483-
,{{ datavault4dbt.ghost_record_per_datatype(column_name=column.name, datatype=column.dtype, ghost_record_type='unknown') }}
483+
,{{ datavault4dbt.ghost_record_per_datatype(column_name=column.name, datatype=column.data_type, ghost_record_type='unknown') }}
484484
{%- endfor -%}
485485

486486
{%- endif -%}
@@ -510,7 +510,7 @@ unknown_values AS (
510510
{%- set prejoin_extract_cols_lower = prejoin['extract_columns']|map('lower')|list -%}
511511
{%- set prejoin_col_index = prejoin_extract_cols_lower.index(column.name|lower) -%}
512512
{% if var('datavault4dbt.show_debug_logs', false) %}{{ log('column found? yes, for column: ' ~ column.name, false) }}{% endif %}
513-
,{{ datavault4dbt.ghost_record_per_datatype(column_name=column.name, datatype=column.dtype, ghost_record_type='unknown', alias=prejoin['aliases'][prejoin_col_index]) }}
513+
,{{ datavault4dbt.ghost_record_per_datatype(column_name=column.name, datatype=column.data_type, ghost_record_type='unknown', alias=prejoin['aliases'][prejoin_col_index]) }}
514514
{%- endif -%}
515515

516516
{%- endfor -%}
@@ -546,7 +546,7 @@ error_values AS (
546546
{%- if columns_without_excluded_columns is defined and columns_without_excluded_columns| length > 0 -%}
547547
{# Generating Ghost Records for Source Columns #}
548548
{%- for column in columns_without_excluded_columns %}
549-
,{{ datavault4dbt.ghost_record_per_datatype(column_name=column.name, datatype=column.dtype, ghost_record_type='error') }}
549+
,{{ datavault4dbt.ghost_record_per_datatype(column_name=column.name, datatype=column.data_type, ghost_record_type='error') }}
550550
{%- endfor -%}
551551

552552
{%- endif -%}
@@ -576,7 +576,7 @@ error_values AS (
576576
{%- set prejoin_extract_cols_lower = prejoin['extract_columns']|map('lower')|list -%}
577577
{%- set prejoin_col_index = prejoin_extract_cols_lower.index(column.name|lower) -%}
578578
{% if var('datavault4dbt.show_debug_logs', false) %}{{ log('column found? yes, for column: ' ~ column.name, false) }}{% endif %}
579-
,{{ datavault4dbt.ghost_record_per_datatype(column_name=column.name, datatype=column.dtype, ghost_record_type='error', alias=prejoin['aliases'][prejoin_col_index]) }}
579+
,{{ datavault4dbt.ghost_record_per_datatype(column_name=column.name, datatype=column.data_type, ghost_record_type='error', alias=prejoin['aliases'][prejoin_col_index]) }}
580580
{%- endif -%}
581581

582582
{%- endfor -%}

‎macros/supporting/ghost_record_per_datatype.sql‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -690,27 +690,53 @@
690690

691691
{%- set unknown_value__STRING = var('datavault4dbt.unknown_value__STRING', '(unknown)') -%}
692692
{%- set error_value__STRING = var('datavault4dbt.error_value__STRING', '(error)') -%}
693+
{%- set unknown_value_alt__STRING = var('datavault4dbt.unknown_value_alt__STRING', 'u') -%}
694+
{%- set error_value_alt__STRING = var('datavault4dbt.error_value_alt__STRING', 'e') -%}
693695

694696
{%- set hash_default_values = fromjson(datavault4dbt.hash_default_values(hash_function=datavault4dbt.hash_method())) -%}
695697
{%- set unknown_key = hash_default_values['unknown_key'] -%}
696698
{%- set error_key = hash_default_values['error_key'] -%}
697699

698700
{%- set datatype = datatype | string | upper | trim -%}
699701

702+
{#- Extract the declared length of sized character types (e.g. VARCHAR(50) -> 50), if present. -#}
703+
{%- set char_length = none -%}
704+
{%- if (datatype.startswith('VARCHAR') or datatype.startswith('CHAR')) and '(' in datatype -%}
705+
{%- set char_length = datatype.split('(')[1].split(')')[0] | int -%}
706+
{%- endif -%}
707+
700708
{%- if ghost_record_type == 'unknown' -%}
701709
{%- if 'TIMESTAMP' in datatype %} CAST({{ datavault4dbt.string_to_timestamp(timestamp_format, beginning_of_all_times) }} AS {{ datatype }}) as {{ alias }}
702710
{%- elif datatype == 'DATE' %} CAST('{{ beginning_of_all_times_date }}' AS DATE) as {{ alias }}
703-
{%- elif datatype in ['VARCHAR', 'STRING', 'TEXT'] %} '{{ unknown_value__STRING }}' as {{ alias }}
704-
{%- elif datatype in ['INTEGER', 'INT', 'BIGINT', 'SMALLINT', 'TINYINT', 'DOUBLE', 'FLOAT', 'REAL', 'DECIMAL'] %} CAST({{ unknown_value__numeric }} as {{ datatype }}) as {{ alias }}
711+
{%- elif datatype.startswith('TIME') -%}
712+
{%- if 'WITH TIME ZONE' in datatype %} CAST('00:00:01+00:00' AS {{ datatype }}) as {{ alias }}
713+
{%- else %} CAST('00:00:01' AS {{ datatype }}) as {{ alias }}
714+
{%- endif -%}
715+
{%- elif datatype.startswith('VARCHAR') or datatype.startswith('CHAR') -%}
716+
{%- if char_length is not none and char_length < unknown_value__STRING | length %} CAST('{{ unknown_value_alt__STRING }}' as {{ datatype }}) as {{ alias }}
717+
{%- else %} CAST('{{ unknown_value__STRING }}' as {{ datatype }}) as {{ alias }}
718+
{%- endif -%}
719+
{%- elif datatype in ['STRING', 'TEXT'] %} '{{ unknown_value__STRING }}' as {{ alias }}
720+
{%- elif datatype.startswith('DECIMAL') or datatype in ['INTEGER', 'INT', 'BIGINT', 'SMALLINT', 'TINYINT', 'DOUBLE', 'FLOAT', 'REAL'] %} CAST({{ unknown_value__numeric }} as {{ datatype }}) as {{ alias }}
705721
{%- elif datatype == 'BOOLEAN' %} CAST(FALSE as BOOLEAN) as {{ alias }}
722+
{%- elif 'BINARY' in datatype %} from_hex({{ datavault4dbt.as_constant(column_str=unknown_key) }}) as {{ alias }}
706723
{%- else %} CAST(NULL as {{ datatype }}) as {{ alias }}
707724
{% endif %}
708725
{%- elif ghost_record_type == 'error' -%}
709726
{%- if 'TIMESTAMP' in datatype %} CAST({{ datavault4dbt.string_to_timestamp(timestamp_format, end_of_all_times) }} AS {{ datatype }}) as {{ alias }}
710727
{%- elif datatype == 'DATE' %} CAST('{{ end_of_all_times_date }}' AS DATE) as {{ alias }}
711-
{%- elif datatype in ['VARCHAR', 'STRING', 'TEXT'] %} '{{ error_value__STRING }}' as {{ alias }}
712-
{%- elif datatype in ['INTEGER', 'INT', 'BIGINT', 'SMALLINT', 'TINYINT', 'DOUBLE', 'FLOAT', 'REAL', 'DECIMAL'] %} CAST({{ error_value__numeric }} as {{ datatype }}) as {{ alias }}
728+
{%- elif datatype.startswith('TIME') -%}
729+
{%- if 'WITH TIME ZONE' in datatype %} CAST('23:59:59+00:00' AS {{ datatype }}) as {{ alias }}
730+
{%- else %} CAST('23:59:59' AS {{ datatype }}) as {{ alias }}
731+
{%- endif -%}
732+
{%- elif datatype.startswith('VARCHAR') or datatype.startswith('CHAR') -%}
733+
{%- if char_length is not none and char_length < error_value__STRING | length %} CAST('{{ error_value_alt__STRING }}' as {{ datatype }}) as {{ alias }}
734+
{%- else %} CAST('{{ error_value__STRING }}' as {{ datatype }}) as {{ alias }}
735+
{%- endif -%}
736+
{%- elif datatype in ['STRING', 'TEXT'] %} '{{ error_value__STRING }}' as {{ alias }}
737+
{%- elif datatype.startswith('DECIMAL') or datatype in ['INTEGER', 'INT', 'BIGINT', 'SMALLINT', 'TINYINT', 'DOUBLE', 'FLOAT', 'REAL'] %} CAST({{ error_value__numeric }} as {{ datatype }}) as {{ alias }}
713738
{%- elif datatype == 'BOOLEAN' %} CAST(FALSE as BOOLEAN) as {{ alias }}
739+
{%- elif 'BINARY' in datatype %} from_hex({{ datavault4dbt.as_constant(column_str=error_key) }}) as {{ alias }}
714740
{%- else %} CAST(NULL as {{ datatype }}) as {{ alias }}
715741
{% endif %}
716742
{%- else -%}

0 commit comments

Comments
 (0)