Skip to content
This repository was archived by the owner on Sep 16, 2025. It is now read-only.

Commit 81a471d

Browse files
navadotanyshak
navado
and
tanyshak
authored
Clean sql (#18)
* deleted extra spaces in compiled models * added target_table_alias * fixed exclude_columns SQL #16 * v1.5.23 --------- Co-authored-by: Tanya Shemet <tanyshak@gmail.com>
1 parent 4145bda commit 81a471d

15 files changed

Lines changed: 99 additions & 100 deletions

File tree

‎dbt/adapters/upsolver/__version__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
version = "1.5.22"
1+
version = "1.5.23"

‎dbt/adapters/upsolver/impl.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,8 @@ def separate_options(self, config_options, source):
7373
return job_options, source_options
7474

7575
def render_option_from_dict(self, option_value):
76-
res = []
7776
try:
77+
res = []
7878
for key, value in option_value.items():
7979
item = [f'{key}=']
8080
if isinstance(value, list):
@@ -86,16 +86,16 @@ def render_option_from_dict(self, option_value):
8686
res.append(''.join(item))
8787
return f"({' ,'.join(res)})"
8888
except Exception:
89-
raise dbt.exceptions.ParsingError(f"Error while parsing value: {value}")
89+
raise dbt.exceptions.ParsingError(f"Error while parsing value: {value}. Expected type: dictionary")
9090

9191
def render_option_from_list(self, option_value):
9292
try:
93-
if not isinstance(option_value, str):
93+
if isinstance(option_value, list) and len(option_value) > 1:
9494
return tuple(i for i in option_value)
9595
else:
96-
return f"('{option_value}')"
96+
return f"('{''.join(option_value)}')"
9797
except Exception:
98-
raise dbt.exceptions.ParsingError(f"Error while parsing value: {value}")
98+
raise dbt.exceptions.ParsingError(f"Error while parsing value: {value}. Expected type: list of strings")
9999

100100
@available
101101
def enrich_options(self, config_options, source, options_type):

‎dbt/include/upsolver/macros/materializations/incremental/create_copy_job.sql‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,19 @@
11
{% macro get_create_copy_job_sql(job_identifier, sql, into_relation, sync, options, source, target_type) -%}
22

3-
{% set connection_identifier = adapter.get_connection_from_sql(sql) %}
4-
{% set job_options, source_options = adapter.separate_options(options, source) %}
3+
{%- set connection_identifier = adapter.get_connection_from_sql(sql) -%}
4+
{%- set job_options, source_options = adapter.separate_options(options, source) -%}
55
{%- if target_type != 'datalake' -%}
6-
{% set target_options = adapter.enrich_options(options, target_type, 'target_options') %}
7-
{% set target_type = target_type %}
6+
{%- set target_options = adapter.enrich_options(options, target_type, 'target_options') -%}
7+
{%- set target_type = target_type -%}
88
{%- else -%}
9-
{% set target_options = {} %}
10-
{% set target_type = '' %}
9+
{%- set target_options = {} -%}
10+
{%- set target_type = '' -%}
1111
{%- endif -%}
1212

13-
CREATE
14-
{% if sync %}
15-
SYNC
16-
{% endif %}
13+
CREATE {{''}}
14+
{%- if sync -%}
15+
SYNC {{''}}
16+
{%- endif -%}
1717
JOB {{job_identifier}}
1818
{{ render_options(job_options, 'create') }}
1919
{{ render_options(target_options, 'create') }}

‎dbt/include/upsolver/macros/materializations/incremental/create_insert_job.sql‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
11
{% macro get_create_insert_job_sql(job_identifier, into_relation, sync, options, map_columns_by_name, target_type) -%}
22

3-
{% set enriched_options = adapter.enrich_options(options, target_type, 'transformation_options') %}
3+
{%- set enriched_options = adapter.enrich_options(options, target_type, 'transformation_options') -%}
44
{%- if target_type == 'datalake' -%}
5-
{% set target_type = '' %}
5+
{%- set target_type = '' -%}
66
{%- endif -%}
77

8-
CREATE
9-
{% if sync %}
10-
SYNC
11-
{% endif %}
12-
JOB {{job_identifier}}
8+
CREATE {{''}}
9+
{%- if sync -%}
10+
SYNC {{''}}
11+
{%- endif -%}
12+
JOB {{job_identifier}}
1313
{{ render_options(enriched_options, 'create') }}
1414
AS INSERT INTO {{target_type}} {{into_relation}}
15-
{% if map_columns_by_name %}
15+
{%- if map_columns_by_name %}
1616
MAP_COLUMNS_BY_NAME
17-
{% endif %}
17+
{%- endif %}
1818
{{sql}}
1919

2020
{%- endmacro %}

‎dbt/include/upsolver/macros/materializations/incremental/create_merge_job.sql‎

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,29 +3,27 @@
33
{% set enriched_options = adapter.enrich_options(options, target_type, 'transformation_options') %}
44

55
{%- if target_type == 'datalake' -%}
6-
{% set target_type = '' %}
6+
{%- set target_type = '' -%}
77
{%- endif -%}
88

9-
CREATE
10-
{% if sync %}
11-
SYNC
12-
{% endif %}
9+
CREATE {{''}}
10+
{%- if sync -%}
11+
SYNC {{''}}
12+
{%- endif -%}
1313
JOB {{ job_identifier }}
1414
{{ render_options(enriched_options, 'create') }}
1515
AS MERGE INTO {{ target_type }} {{ into_relation }} AS target
16-
USING (
17-
{{ sql }}
18-
)
19-
{% if primary_key %}
16+
USING ( {{- sql }} )
17+
{% if primary_key -%}
2018
source ON (
21-
{% for item in primary_key %}
19+
{%- for item in primary_key %}
2220
target.{{ item['field'] }} = source.{{ item['field'] }}
23-
{% endfor %}
21+
{%- endfor -%}
2422
)
25-
{% endif %}
23+
{%- endif -%}
2624
{% if delete_condition %}
27-
WHEN MATCHED AND {{ delete_condition}} THEN DELETE
28-
{% endif %}
25+
WHEN MATCHED AND {{ delete_condition}} THEN DELETE
26+
{%- endif %}
2927
WHEN MATCHED THEN REPLACE
3028
WHEN NOT MATCHED THEN INSERT MAP_COLUMNS_BY_NAME
3129
{%- endmacro %}

‎dbt/include/upsolver/macros/materializations/incremental/incremental.sql‎

Lines changed: 30 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,18 @@
22

33
{%- set identifier = model['alias'] -%}
44
{%- set model_config = model['config'] -%}
5-
{% set incremental_strategy = adapter.get(model_config, 'incremental_strategy', False) %}
6-
{% set sync = adapter.get(model_config, 'sync', False) %}
7-
{% set options = adapter.get(model_config, 'options', {}) %}
8-
{% set source = adapter.get(model_config, 'source') %}
9-
{% set target_type = adapter.get(model_config, 'target_type', 'datalake').lower() %}
10-
{% set target_schema = adapter.get(model_config, 'target_schema', schema) %}
11-
{% set delete_condition = adapter.get(model_config, 'delete_condition', False) %}
12-
{% set partition_by = adapter.get(model_config, 'partition_by', []) %}
13-
{% set primary_key = adapter.get(model_config, 'primary_key', []) %}
14-
{% set map_columns_by_name = adapter.get(model_config, 'map_columns_by_name', False) %}
15-
{% set job_identifier = identifier + '_job' %}
5+
{%- set incremental_strategy = adapter.get(model_config, 'incremental_strategy', False) -%}
6+
{%- set sync = adapter.get(model_config, 'sync', False) -%}
7+
{%- set options = adapter.get(model_config, 'options', {}) -%}
8+
{%- set source = adapter.get(model_config, 'source') -%}
9+
{%- set target_type = adapter.get(model_config, 'target_type', 'datalake').lower() -%}
10+
{%- set target_schema = adapter.get(model_config, 'target_schema', schema) -%}
11+
{%- set target_table_alias = adapter.get(model_config, 'target_table_alias', identifier) -%}
12+
{%- set delete_condition = adapter.get(model_config, 'delete_condition', False) -%}
13+
{%- set partition_by = adapter.get(model_config, 'partition_by', []) -%}
14+
{%- set primary_key = adapter.get(model_config, 'primary_key', []) -%}
15+
{%- set map_columns_by_name = adapter.get(model_config, 'map_columns_by_name', False) -%}
16+
{%- set job_identifier = identifier + '_job' %}
1617

1718
{%- set old_relation = adapter.get_relation(identifier=job_identifier,
1819
schema=schema,
@@ -29,7 +30,7 @@
2930

3031

3132
{% if target_type == 'datalake' %}
32-
{%- set table_relation = api.Relation.create(identifier=identifier,
33+
{%- set table_relation = api.Relation.create(identifier=target_table_alias,
3334
schema=schema,
3435
database=database,
3536
type='table') -%}
@@ -39,42 +40,42 @@
3940
{%- endcall -%}
4041
{%- else -%}
4142
{% set target_connection = adapter.require(model_config, 'target_connection') %}
42-
{%- set into_relation = target_connection + '.' + target_schema + '.' + identifier -%}
43+
{%- set into_relation = target_connection + '.' + target_schema + '.' + target_table_alias -%}
4344
{%- endif %}
4445

45-
{% if old_relation %}
46-
{% call statement('main') -%}
46+
{%- if old_relation -%}
47+
{%- call statement('main') -%}
4748
{{ get_alter_job_sql(job_identifier, options, incremental_strategy, source) }}
4849
{%- endcall %}
49-
{% else %}
50-
{% call statement('main') -%}
51-
{% if incremental_strategy == 'merge' %}
50+
{%- else -%}
51+
{%- call statement('main') -%}
52+
{%- if incremental_strategy == 'merge' -%}
5253
{{ get_create_merge_job_sql(job_identifier, into_relation, sync,
5354
options, primary_key, delete_condition,
5455
target_type) }}
55-
{% elif incremental_strategy == 'insert' %}
56+
{%- elif incremental_strategy == 'insert' -%}
5657
{{ get_create_insert_job_sql(job_identifier,
5758
into_relation, sync, options,
5859
map_columns_by_name, target_type) }}
5960

60-
{% else %}
61+
{%- else -%}
6162
{{ get_create_copy_job_sql(job_identifier, sql,
6263
into_relation, sync, options, source,
6364
target_type) }}
6465

65-
{% endif %}
66-
{%- endcall %}
67-
{%- endif %}
66+
{%- endif -%}
67+
{%- endcall -%}
68+
{%- endif -%}
6869

69-
{% do persist_docs(target_relation, model) %}
70-
{% do persist_docs(table_relation, model) %}
70+
{%- do persist_docs(target_relation, model) -%}
71+
{%- do persist_docs(table_relation, model) -%}
7172

7273
{{ run_hooks(post_hooks, inside_transaction=False) }}
7374
{{ run_hooks(post_hooks, inside_transaction=True) }}
7475

75-
{% if target_type == 'datalake' %}
76+
{%- if target_type == 'datalake' -%}
7677
{{ return({'relations': [target_relation, table_relation]}) }}
77-
{% else %}
78+
{%- else -%}
7879
{{ return({'relations': [target_relation]}) }}
79-
{%- endif %}
80-
{% endmaterialization %}
80+
{%- endif -%}
81+
{%- endmaterialization -%}

‎dbt/include/upsolver/macros/materializations/utils/ater_job.sql‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
{% macro get_alter_job_sql(job_identifier, options, incremental_strategy, source) -%}
22

3-
{% if incremental_strategy %}
4-
{% set enriched_options = adapter.enrich_options(options, 'datalake', 'transformation_options') %}
5-
{% else %}
6-
{% set enriched_options, _ = adapter.separate_options(options, source) %}
7-
{% endif %}
8-
{% set enriched_editable_options = adapter.filter_options(enriched_options, 'editable') %}
3+
{%- if incremental_strategy -%}
4+
{%- set enriched_options = adapter.enrich_options(options, 'datalake', 'transformation_options') -%}
5+
{%- else -%}
6+
{%- set enriched_options, _ = adapter.separate_options(options, source) -%}
7+
{%- endif -%}
8+
{%- set enriched_editable_options = adapter.filter_options(enriched_options, 'editable') -%}
99

1010
ALTER JOB {{job_identifier}}
1111
{{ render_options(enriched_editable_options, 'alter') }}

‎dbt/include/upsolver/macros/materializations/utils/create_table.sql‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,24 +7,24 @@
77
{%- set columns_with_types = adapter.get_columns_names_with_types(partition_by + primary_key) -%}
88
{%- set columns_partitioned_by = adapter.get_columns_names(partition_by) -%}
99
{%- set columns_primary_key = adapter.get_columns_names(primary_key) -%}
10-
{% set enriched_options = adapter.enrich_options(options, 'datalake', 'target_options') %}
11-
{% set enriched_editable_options = adapter.filter_options(enriched_options, 'editable') %}
10+
{%- set enriched_options = adapter.enrich_options(options, 'datalake', 'target_options') -%}
11+
{%- set enriched_editable_options = adapter.filter_options(enriched_options, 'editable') -%}
1212

13-
{% if old_relation %}
13+
{%- if old_relation -%}
1414
ALTER TABLE {{target_relation}}
15-
{{ render_options(enriched_editable_options, 'alter') }}
16-
{% else %}
15+
{{ render_options(enriched_editable_options, 'alter') }}
16+
{%- else -%}
1717
CREATE TABLE {{ target_relation }}
1818
({{ columns_with_types }})
19-
{% if partition_by %}
20-
PARTITIONED BY
19+
{%- if partition_by %}
20+
PARTITIONED BY
2121
{{ columns_partitioned_by }}
22-
{% endif %}
23-
{% if primary_key %}
24-
PRIMARY KEY
22+
{%- endif -%}
23+
{%- if primary_key %}
24+
PRIMARY KEY
2525
{{ columns_primary_key }}
26-
{% endif %}
26+
{%- endif %}
2727
{{ render_options(enriched_options, 'create') }}
28-
{% endif %}
28+
{%- endif -%}
2929

3030
{%- endmacro %}

‎dbt/include/upsolver/macros/materializations/utils/render_options.sql‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
1-
{% macro render_options(options, statement) -%}
1+
{%- macro render_options(options, statement) -%}
22

33
{%- if statement == 'alter' -%}
4-
{% set key = 'SET' %}
4+
{%- set key = 'SET' -%}
55
{%- endif -%}
6-
{%- for k, v in options.items() -%}
7-
{% set value = v['value'] %}
8-
{% if v['type'] == 'text' %}
6+
{%- for k, v in options.items() %}
7+
{%- set value = v['value'] -%}
8+
{%- if v['type'] == 'text' %}
99
{{key}} {{k}} = '{{ value }}'
10-
{% elif v['type'] == 'identifier' %}
10+
{%- elif v['type'] == 'identifier' %}
1111
{{key}} {{k}} = "{{ value }}"
12-
{% else %}
12+
{%- else %}
1313
{{key}} {{k}} = {{ value }}
1414
{%- endif -%}
1515
{%- endfor %}

examples/upsert_records_new/models/dell_test2.sql renamed to examples/upsert_records_new/models/dell_test.sql

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,5 +11,5 @@
1111

1212
SELECT
1313
*
14-
FROM {{ source('upsert_records_new', 'orders_raw_data_for_upsert_2') }}
14+
FROM {{ source('upsert_records_new', 'orders_raw_data_for_upsert') }}
1515
WHERE $event_time BETWEEN run_start_time() AND run_end_time()

0 commit comments

Comments
 (0)