Skip to content
This repository was archived by the owner on Sep 16, 2025. It is now read-only.

Commit ce69d21

Browse files
navadotanyshak
navado
and
tanyshak
authored
Fix Delete Condition (#12)
* removed delete_placeholder to use delete_condition properly * added delete_condition example * fixed list options * fixed options render * Increment version for build * fixed the dictionary of options * fixed the missing commas * added heartbeat_table to options * added parse_json_columns * Fix typo in macro name --------- Co-authored-by: Tanya Shemet <tanyshak@gmail.com>
1 parent 1ab62a1 commit ce69d21

10 files changed

Lines changed: 70 additions & 45 deletions

File tree

‎dbt/adapters/upsolver/__version__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
version = "0.2.0"
1+
version = "0.2.1"

‎dbt/adapters/upsolver/impl.py‎

Lines changed: 12 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import agate
1313
import datetime
1414
import re
15+
import dbt
1516

1617
logger = AdapterLogger("Upsolver")
1718
LIST_RELATION_MACRO_NAME = "list_relation_without_caching"
@@ -44,10 +45,12 @@ def drop_schema(self, relation: UpsolverRelation) -> None:
4445

4546
@available
4647
def get_connection_from_sql(self, sql):
47-
connection_identifier = re.search('"(.*)"', sql).group().split('.')[2] \
48-
.translate(str.maketrans({'\"':'', '\'':''}))
49-
50-
return connection_identifier
48+
try:
49+
connection_identifier = re.search('"(.*)"', sql).group().split('.')[2] \
50+
.translate(str.maketrans({'\"':'', '\'':''}))
51+
return connection_identifier
52+
except Exception:
53+
raise dbt.exceptions.ParsingError(f"Error while parsing connection name from sql:\n{sql}")
5154

5255
@available
5356
def get_columns_names_with_types(self, list_dict):
@@ -77,8 +80,11 @@ def enrich_options(self, config_options, source, options_type):
7780
for option, value in config_options.items():
7881
find_value = options.get(option.lower(), None)
7982
if find_value:
80-
if options[option.lower()]['type'] == 'list' and isinstance(value, str):
81-
value = f"('{value}')"
83+
if options[option.lower()]['type'] == 'list':
84+
if not isinstance(value, str):
85+
value = tuple(i for i in value)
86+
else:
87+
value = f"('{value}')"
8288
enriched_options[option] = find_value
8389
enriched_options[option]['value'] = value
8490
else:
@@ -103,16 +109,6 @@ def get_options(self, source, options_type):
103109
options = Copy_options[source.lower()][options_type]
104110
return options
105111

106-
@available
107-
def get_delete_placeholder(self, sql, delete_condition):
108-
if delete_condition:
109-
delete_placeholder= re.search('(nettotal < 0 [as|AS,\s]*\w*)', sql)[1] \
110-
.lower().replace(" ", "") \
111-
.replace(f"{delete_condition.lower().replace(' ', '')}as", "")
112-
return delete_placeholder
113-
else:
114-
return False
115-
116112
def list_relations_without_caching(
117113
self,
118114
schema_relation: UpsolverRelation,

‎dbt/adapters/upsolver/options/copy_options.py‎

Lines changed: 27 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -16,34 +16,31 @@
1616
"comment": {"type": "text", "editable": True, "optional": True}
1717
}
1818
},
19-
"my_sql": {
19+
"mysql": {
2020
"source_options": {
21-
"table_include_list": {"type": "text", "editable": True, "optional": True},
22-
"column_exclude_list": {"type": "text", "editable": True, "optional": True}
21+
"table_include_list": {"type": "list", "editable": True, "optional": True},
22+
"column_exclude_list": {"type": "list", "editable": True, "optional": True}
2323
},
2424
"job_options": {
25-
"skip_snapshots": {"value": "text", "editable": True, "optional": True},
25+
"skip_snapshots": {"type": "boolean", "editable": True, "optional": True},
2626
"end_at": {"type": "value", "editable": True, "optional": True},
2727
"compute_cluster": {"type": "identifier", "editable": True, "optional": True},
2828
"comment": {"type": "text", "editable": True, "optional": True}
2929
}
3030
},
31-
"postgre_sql": {
31+
"postgres": {
3232
"source_options": {
33-
"bucket": {"type": "text", "editable": False, "optional": False},
34-
"prefix": {"type": "text", "editable": False, "optional": True}
33+
"table_include_list": {"type": "list", "editable": False, "optional": False},
34+
"column_exclude_list": {"type": "list", "editable": False, "optional": True}
3535
},
3636
"job_options": {
37-
"table_include_list": {"type": "text", "editable": False, "optional": True},
37+
"heartbeat_table": {"type": "text", "editable": False, "optional": True},
3838
"skip_snapshots": {"type": "boolean", "editable": False, "optional": True},
39-
"column_exclude_list": {"type": "text", "editable": False, "optional": True},
4039
"publication_name": {"type": "text", "editable": False, "optional": False},
41-
"file_pattern": {"type": "text", "editable": False, "optional": True},
42-
"delete_files_after_load": {"type": "boolean", "editable": False, "optional": True},
4340
"end_at": {"type": "value", "editable": True, "optional": True},
4441
"compute_cluster": {"type": "identifier", "editable": True, "optional": True},
45-
"run_parallelism": {"type": "integer", "editable": True, "optional": True},
46-
"comment": {"type": "text", "editable": True, "optional": True}
42+
"comment": {"type": "text", "editable": True, "optional": True},
43+
"parse_json_columns": {"type": "boolean", "editable": False, "optional": False}
4744
}
4845
},
4946
"s3": {
@@ -60,5 +57,21 @@
6057
"compression": {"type": "value", "editable": False, "optional": True},
6158
"comment": {"type": "text", "editable": True, "optional": True}
6259
}
63-
}
60+
},
61+
"kinesis": {
62+
"source_options": {
63+
"stream": {"type": "text", "editable": False, "optional": False}
64+
},
65+
"job_options": {
66+
"reader_shards": {"type": "integer", "editable": True, "optional": True},
67+
"store_raw_data": {"type": "boolean", "editable": False, "optional": True},
68+
"start_from": {"type": "value", "editable": False, "optional": True},
69+
"end_at": {"type": "value", "editable": False, "optional": True},
70+
"compute_cluster": {"type": "identifier", "editable": True, "optional": True},
71+
"run_parallelism": {"type": "integer", "editable": False, "optional": True},
72+
"content_type": {"type": "value", "editable": True, "optional": True},
73+
"compression": {"type": "value", "editable": False, "optional": True},
74+
"comment": {"type": "text", "editable": True, "optional": True}
75+
}
76+
}
6477
}

‎dbt/adapters/upsolver/options/table_options.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"storage_location": {"type": "text", "editable": False, "optional": True},
55
"compute_cluster": {"type": "identifier", "editable": True, "optional": True},
66
"compression": {"type": "value", "editable": True, "optional": True},
7-
"compaction_processes": {"type": "list", "editable": True, "optional": True},
7+
"compaction_processes": {"type": "integer", "editable": True, "optional": True},
88
"disable_compaction": {"type": "boolean", "editable": True, "optional": True},
99
"retention_date_partition": {"type": "text", "editable": True, "optional": True},
1010
"table_data_retention": {"type": "text", "editable": True, "optional": True},

‎dbt/include/upsolver/macros/materializations/connection.sql‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,6 @@
1818
{{ run_hooks(pre_hooks, inside_transaction=False) }}
1919
{{ run_hooks(pre_hooks, inside_transaction=True) }}
2020

21-
{{ log("Options: " ~ connection_options ) }}
22-
{{ log("Enriched options: " ~ enriched_options ) }}
23-
{{ log("Enriched options: " ~ enriched_options ) }}
24-
25-
2621
{% if old_relation %}
2722
{% call statement('main') %}
2823
ALTER {{ connection_type }} CONNECTION {{target_relation.identifier}}

‎dbt/include/upsolver/macros/materializations/incremental/create_insert_job.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
{% macro get_create_incert_job_sql(job_identifier, table, sync, options, map_columns_by_name) -%}
1+
{% macro get_create_insert_job_sql(job_identifier, table, sync, options, map_columns_by_name) -%}
22

33
{% set enriched_options = adapter.enrich_options(options, 'upsolver_data_lake', 'transformation_options') %}
44

‎dbt/include/upsolver/macros/materializations/incremental/create_merge_job.sql‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
{% macro get_create_merge_job_sql(job_identifier, table, sync, options, primary_key, delete_condition) -%}
22

33
{% set enriched_options = adapter.enrich_options(options, 'upsolver_data_lake', 'transformation_options') %}
4-
{% set delete_placeholder = adapter.get_delete_placeholder(sql, delete_condition) %}
54

65
CREATE
76
{% if sync %}
@@ -20,12 +19,9 @@
2019
{% endfor %}
2120
)
2221
{% endif %}
23-
{% if delete_placeholder %}
24-
WHEN MATCHED AND {{ delete_placeholder }} THEN DELETE
22+
{% if delete_condition %}
23+
WHEN MATCHED AND {{ delete_condition}} THEN DELETE
2524
{% endif %}
2625
WHEN MATCHED THEN REPLACE
2726
WHEN NOT MATCHED THEN INSERT MAP_COLUMNS_BY_NAME
28-
{% if delete_placeholder %}
29-
EXCEPT {{ delete_placeholder }}
30-
{% endif %}
3127
{%- endmacro %}

‎dbt/include/upsolver/macros/materializations/incremental/incremental.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242
{{ get_create_merge_job_sql(job_identifier, table_relation, sync,
4343
options, primary_key, delete_condition) }}
4444
{% elif incremental_strategy == 'insert' %}
45-
{{ get_create_incert_job_sql(job_identifier,
45+
{{ get_create_insert_job_sql(job_identifier,
4646
table_relation, sync, options,
4747
map_columns_by_name) }}
4848

‎examples/upsert_records_new/models/dell_test2.sql‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{{ config( materialized='incremental',
2+
incremental_strategy='merge',
3+
map_columns_by_name=True,
4+
sync=True,
5+
options={'START_FROM': 'NOW',
6+
'ADD_MISSING_COLUMNS': True,
7+
'RUN_INTERVAL': '1 MINUTE'},
8+
primary_key=[{'field':'orderid', 'type':'string'}],
9+
delete_condition='nettotal > 1000' )
10+
}}
11+
12+
SELECT
13+
*
14+
FROM {{ source('upsert_records_new', 'orders_raw_data_for_upsert_2') }}
15+
WHERE $event_time BETWEEN run_start_time() AND run_end_time()

‎examples/upsert_records_new/models/sources.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
version: 2
2+
3+
sources:
4+
- name: upsert_records_new
5+
database: default_glue_catalog
6+
schema: database_16e61b
7+
tables:
8+
- name: insert_orders_upsert_2
9+
- name: merge_orders_upsert_2
10+
- name: orders_raw_data_for_upsert_2

0 commit comments

Comments
 (0)