feat: add overwrite_src_cols parameter to derived_columns

Adds a new per-derived-column boolean parameter overwrite_src_cols
(default: false). When true, the columns listed in src_cols_required
are suppressed from the derived_columns CTE SELECT onward, so the
source column is replaced by the alias rather than duplicated alongside
it. Fixes the case where renaming a column with special characters
(e.g. German umlauts on Databricks) resulted in both the original and
the alias appearing in all downstream CTEs.

Implemented via a new extract_overwrite_columns() helper macro in
stage_processing_macros.sql. The helper is called before
derived_columns_datatypes() to avoid the in-place dict mutation that
strips custom keys from the columns dict.

Author: pkumar-data

macros/staging/bigquery/stage.sql

@@ -156,9 +156,12 @@
 {%- set derived_columns_to_select = datavault4dbt.process_columns_to_select(source_and_derived_column_names, hashed_column_names) | unique | list -%}
 
 {%- if datavault4dbt.is_something(derived_columns) %}
+  {%- set overwrite_src_columns = datavault4dbt.extract_overwrite_columns(derived_columns) -%}
   {#- Getting Data types for derived columns with detection from source relation -#}
   {%- set derived_columns_with_datatypes = datavault4dbt.derived_columns_datatypes(derived_columns, source_relation) -%}
   {%- set derived_columns_with_datatypes_DICT = fromjson(derived_columns_with_datatypes) -%}
+{%- else -%}
+  {%- set overwrite_src_columns = [] -%}
 {%- endif -%}
 {#- Select hashing algorithm -#}
 
@@ -356,6 +359,7 @@ prejoined_columns AS (
 derived_columns AS (
 
   {%- set final_columns_to_select = datavault4dbt.process_columns_to_select(final_columns_to_select, derived_column_names) -%}
+  {%- set final_columns_to_select = datavault4dbt.process_columns_to_select(final_columns_to_select, overwrite_src_columns) -%}
 
   SELECT
   {% if final_columns_to_select | length > 0 -%}

macros/staging/databricks/stage.sql

@@ -156,9 +156,12 @@
 {%- set derived_columns_to_select = datavault4dbt.process_columns_to_select(source_and_derived_column_names, hashed_column_names) | unique | list -%}
 
 {%- if datavault4dbt.is_something(derived_columns) %}
+  {%- set overwrite_src_columns = datavault4dbt.extract_overwrite_columns(derived_columns) -%}
   {#- Getting Data types for derived columns with detection from source relation -#}
   {%- set derived_columns_with_datatypes = datavault4dbt.derived_columns_datatypes(derived_columns, source_relation) -%}
   {%- set derived_columns_with_datatypes_DICT = fromjson(derived_columns_with_datatypes) -%}
+{%- else -%}
+  {%- set overwrite_src_columns = [] -%}
 {%- endif -%}
 {#- Select hashing algorithm -#}
 
@@ -357,6 +360,7 @@ prejoined_columns AS (
 derived_columns AS (
 
   {%- set final_columns_to_select = datavault4dbt.process_columns_to_select(final_columns_to_select, derived_column_names) -%}
+  {%- set final_columns_to_select = datavault4dbt.process_columns_to_select(final_columns_to_select, overwrite_src_columns) -%}
 
   SELECT
 

macros/staging/exasol/stage.sql

@@ -151,9 +151,12 @@
 {%- set derived_columns_to_select = datavault4dbt.process_columns_to_select(source_and_derived_column_names, hashed_column_names) | unique | list -%}
 
 {%- if datavault4dbt.is_something(derived_columns) %}
+  {%- set overwrite_src_columns = datavault4dbt.extract_overwrite_columns(derived_columns) -%}
   {#- Getting Data types for derived columns with detection from source relation -#}
   {%- set derived_columns_with_datatypes = datavault4dbt.derived_columns_datatypes(derived_columns, source_relation) -%}
   {%- set derived_columns_with_datatypes_DICT = fromjson(derived_columns_with_datatypes) -%}
+{%- else -%}
+  {%- set overwrite_src_columns = [] -%}
 {%- endif -%}
 {#- Select hashing algorithm -#}
 
@@ -352,6 +355,7 @@ prejoined_columns AS (
 derived_columns AS (
 
   {%- set final_columns_to_select = datavault4dbt.process_columns_to_select(final_columns_to_select, derived_column_names) -%}
+  {%- set final_columns_to_select = datavault4dbt.process_columns_to_select(final_columns_to_select, overwrite_src_columns) -%}
 
   SELECT
   {% if final_columns_to_select | length > 0 -%}

macros/staging/fabric/stage.sql

@@ -150,9 +150,12 @@
 {%- set derived_columns_to_select = datavault4dbt.process_columns_to_select(source_and_derived_column_names, hashed_column_names) | unique | list -%}
 
 {%- if datavault4dbt.is_something(derived_columns) %}
+  {%- set overwrite_src_columns = datavault4dbt.extract_overwrite_columns(derived_columns) -%}
   {#- Getting Data types for derived columns with detection from source relation -#}
   {%- set derived_columns_with_datatypes = datavault4dbt.derived_columns_datatypes(derived_columns, source_relation) -%}
   {%- set derived_columns_with_datatypes_DICT = fromjson(derived_columns_with_datatypes) -%}
+{%- else -%}
+  {%- set overwrite_src_columns = [] -%}
 {%- endif -%}
 {#- Select hashing algorithm -#}
 
@@ -362,6 +365,7 @@ derived_columns AS (
   {%- endif -%}
 
   {%- set final_columns_to_select = datavault4dbt.process_columns_to_select(final_columns_to_select, exluded_derived_columns) -%}
+  {%- set final_columns_to_select = datavault4dbt.process_columns_to_select(final_columns_to_select, overwrite_src_columns) -%}
 
   SELECT
   {% if final_columns_to_select | length > 0 -%}

macros/staging/oracle/stage.sql

@@ -159,9 +159,12 @@
   {{ log('derived_columns_to select : '~ derived_columns_to_select, false) }}
 
 {%- if datavault4dbt.is_something(derived_columns) %}
+  {%- set overwrite_src_columns = datavault4dbt.extract_overwrite_columns(derived_columns) -%}
   {#- Getting Data types for derived columns with detection from source relation -#}
   {%- set derived_columns_with_datatypes = datavault4dbt.derived_columns_datatypes(derived_columns, source_relation) -%}
   {%- set derived_columns_with_datatypes_DICT = fromjson(derived_columns_with_datatypes) -%}
+{%- else -%}
+  {%- set overwrite_src_columns = [] -%}
 {%- endif -%}
 {#- Select hashing algorithm -#}
 
@@ -363,6 +366,7 @@ prejoined_columns AS (
 derived_columns AS (
 
   {%- set final_columns_to_select = datavault4dbt.process_columns_to_select(final_columns_to_select, derived_column_names) -%}
+  {%- set final_columns_to_select = datavault4dbt.process_columns_to_select(final_columns_to_select, overwrite_src_columns) -%}
 
   SELECT
   {% if final_columns_to_select | length > 0 -%}

macros/staging/postgres/stage.sql

@@ -156,9 +156,12 @@
 {%- set derived_columns_to_select = datavault4dbt.process_columns_to_select(source_and_derived_column_names, hashed_column_names) | unique | list -%}
 
 {%- if datavault4dbt.is_something(derived_columns) %}
+  {%- set overwrite_src_columns = datavault4dbt.extract_overwrite_columns(derived_columns) -%}
   {#- Getting Data types for derived columns with detection from source relation -#}
   {%- set derived_columns_with_datatypes = datavault4dbt.derived_columns_datatypes(derived_columns, source_relation) -%}
   {%- set derived_columns_with_datatypes_DICT = fromjson(derived_columns_with_datatypes) -%}
+{%- else -%}
+  {%- set overwrite_src_columns = [] -%}
 {%- endif -%}
 {#- Select hashing algorithm -#}
 
@@ -357,6 +360,7 @@ prejoined_columns AS (
 derived_columns AS (
 
   {%- set final_columns_to_select = datavault4dbt.process_columns_to_select(final_columns_to_select, derived_column_names) -%}
+  {%- set final_columns_to_select = datavault4dbt.process_columns_to_select(final_columns_to_select, overwrite_src_columns) -%}
 
   SELECT
   {% if final_columns_to_select | length > 0 -%}

macros/staging/redshift/stage.sql

@@ -156,9 +156,12 @@
 {%- set derived_columns_to_select = datavault4dbt.process_columns_to_select(source_and_derived_column_names, hashed_column_names) | unique | list -%}
 
 {%- if datavault4dbt.is_something(derived_columns) %}
+  {%- set overwrite_src_columns = datavault4dbt.extract_overwrite_columns(derived_columns) -%}
   {#- Getting Data types for derived columns with detection from source relation -#}
   {%- set derived_columns_with_datatypes = datavault4dbt.derived_columns_datatypes(derived_columns, source_relation) -%}
   {%- set derived_columns_with_datatypes_DICT = fromjson(derived_columns_with_datatypes) -%}
+{%- else -%}
+  {%- set overwrite_src_columns = [] -%}
 {%- endif -%}
 {#- Select hashing algorithm -#}
 
@@ -357,6 +360,7 @@ prejoined_columns AS (
 derived_columns AS (
 
   {%- set final_columns_to_select = datavault4dbt.process_columns_to_select(final_columns_to_select, derived_column_names) -%}
+  {%- set final_columns_to_select = datavault4dbt.process_columns_to_select(final_columns_to_select, overwrite_src_columns) -%}
 
   SELECT
   {% if final_columns_to_select | length > 0 -%}

macros/staging/snowflake/stage.sql

@@ -160,9 +160,12 @@
   {{ log('derived_columns_to select : '~ derived_columns_to_select, false) }}
 
 {%- if datavault4dbt.is_something(derived_columns) %}
+  {%- set overwrite_src_columns = datavault4dbt.extract_overwrite_columns(derived_columns) -%}
   {#- Getting Data types for derived columns with detection from source relation -#}
   {%- set derived_columns_with_datatypes = datavault4dbt.derived_columns_datatypes(derived_columns, source_relation) -%}
   {%- set derived_columns_with_datatypes_DICT = fromjson(derived_columns_with_datatypes) -%}
+{%- else -%}
+  {%- set overwrite_src_columns = [] -%}
 {%- endif -%}
 {#- Select hashing algorithm -#}
 
@@ -364,6 +367,7 @@ prejoined_columns AS (
 derived_columns AS (
 
   {%- set final_columns_to_select = datavault4dbt.process_columns_to_select(final_columns_to_select, derived_column_names) -%}
+  {%- set final_columns_to_select = datavault4dbt.process_columns_to_select(final_columns_to_select, overwrite_src_columns) -%}
 
   SELECT
   {% if final_columns_to_select | length > 0 -%}

macros/staging/stage_processing_macros.sql

@@ -168,6 +168,30 @@
 {%- endmacro -%}
 
 
+{%- macro extract_overwrite_columns(columns_dict=none) -%}
+{#- Returns all src_cols_required for derived columns where overwrite_src_cols=true.
+    Used to drop the original source column from the derived_columns CTE SELECT list,
+    so only the renamed alias survives (e.g. when renaming a column with special characters). -#}
+
+    {%- set ns = namespace(overwrite_columns = []) -%}
+
+    {%- if columns_dict is mapping -%}
+        {%- for key, value in columns_dict.items() -%}
+            {%- if value is mapping and value.get('overwrite_src_cols', false) and 'src_cols_required' in value -%}
+                {%- if datavault4dbt.is_list(value['src_cols_required']) -%}
+                    {%- set ns.overwrite_columns = ns.overwrite_columns + value['src_cols_required'] -%}
+                {%- else -%}
+                    {%- do ns.overwrite_columns.append(value['src_cols_required']) -%}
+                {%- endif -%}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+
+    {%- do return(ns.overwrite_columns) -%}
+
+{%- endmacro -%}
+
+
 {%- macro process_hash_column_excludes(hash_columns=none, source_columns=none) -%}
 
     {%- set processed_hash_columns = {} -%}

macros/staging/synapse/stage.sql

@@ -149,9 +149,12 @@
 {%- set derived_columns_to_select = datavault4dbt.process_columns_to_select(source_and_derived_column_names, hashed_column_names) | unique | list -%}
 
 {%- if datavault4dbt.is_something(derived_columns) %}
+  {%- set overwrite_src_columns = datavault4dbt.extract_overwrite_columns(derived_columns) -%}
   {#- Getting Data types for derived columns with detection from source relation -#}
   {%- set derived_columns_with_datatypes = datavault4dbt.derived_columns_datatypes(derived_columns, source_relation) -%}
   {%- set derived_columns_with_datatypes_DICT = fromjson(derived_columns_with_datatypes) -%}
+{%- else -%}
+  {%- set overwrite_src_columns = [] -%}
 {%- endif -%}
 {#- Select hashing algorithm -#}
 
@@ -363,6 +366,7 @@ derived_columns AS (
   {%- endif -%}
 
   {%- set final_columns_to_select = datavault4dbt.process_columns_to_select(final_columns_to_select, exluded_derived_columns) -%}
+  {%- set final_columns_to_select = datavault4dbt.process_columns_to_select(final_columns_to_select, overwrite_src_columns) -%}
 
   SELECT
   {% if final_columns_to_select | length > 0 -%}