Skip to content

Commit afa2383

Browse files
fix: expand YAML merge keys in profiles template for dbt-fusion compatibility (#983)
* fix: expand YAML merge keys in profiles template for dbt-fusion compatibility dbt-fusion's Rust YAML parser does not resolve YAML merge keys (<<:), causing 'dbt1005: missing type field in resolved profile output' errors when it tries to parse the 'elementary' profile targets that inherited fields via merge keys from the 'elementary_tests' anchors. This commit: - Replaces the Jinja2 for-loop that generated merge-key-based targets in the 'elementary' profile with explicit full target definitions - Removes the now-unused YAML anchors (&name) from elementary_tests - Preserves identical rendered output (same fields, same values) Fixes fusion CI jobs for snowflake, bigquery, and databricks_catalog. Co-Authored-By: Noy Arie <noyarie1992@gmail.com> * refactor: use Jinja macros to DRY up profiles while avoiding YAML merge keys --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Noy Arie <noyarie1992@gmail.com>
1 parent edf8d50 commit afa2383

File tree

1 file changed

+86
-43
lines changed

1 file changed

+86
-43
lines changed
Lines changed: 86 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,45 @@
1-
elementary_tests:
2-
target: postgres
3-
outputs:
4-
5-
# ── Local targets (plaintext, no secrets needed) ─────────────────
6-
7-
postgres: &postgres
1+
{#- ── Target macros ──────────────────────────────────────────────────
2+
Each target is defined once as a macro so both profiles stay in sync.
3+
We avoid YAML merge keys (<<: *anchor) because dbt-fusion's parser
4+
does not resolve them, causing "dbt1005: missing 'type' field" errors.
5+
-#}
6+
{%- macro postgres_target(schema) %}
7+
postgres:
88
type: postgres
99
host: 127.0.0.1
1010
port: 5432
1111
user: admin
1212
password: admin
1313
dbname: postgres
14-
schema: {{ schema_name }}
14+
schema: {{ schema }}
1515
threads: 32
16+
{%- endmacro %}
1617

17-
clickhouse: &clickhouse
18+
{%- macro clickhouse_target(schema) %}
19+
clickhouse:
1820
type: clickhouse
1921
host: localhost
2022
port: 8123
2123
user: default
2224
password: default
23-
schema: {{ schema_name }}
25+
schema: {{ schema }}
2426
threads: 4
27+
{%- endmacro %}
2528

26-
trino: &trino
29+
{%- macro trino_target(schema) %}
30+
trino:
2731
type: trino
2832
method: none
2933
host: localhost
3034
port: 8086
3135
user: admin
3236
database: iceberg
33-
schema: {{ schema_name }}
37+
schema: {{ schema }}
3438
threads: 4
39+
{%- endmacro %}
3540

36-
dremio: &dremio
41+
{%- macro dremio_target(schema) %}
42+
dremio:
3743
type: dremio
3844
software_host: localhost
3945
port: 9047
@@ -43,128 +49,165 @@ elementary_tests:
4349
datalake: S3Source
4450
root_path: datalake.elementary_ci.{{ schema_name }}
4551
database: elementary_ci
46-
schema: {{ schema_name }}
52+
schema: {{ schema }}
4753
threads: 4
54+
{%- endmacro %}
4855

49-
spark: &spark
56+
{%- macro spark_target(schema) %}
57+
spark:
5058
type: spark
5159
method: thrift
5260
host: 127.0.0.1
5361
port: 10000
5462
user: dbt
55-
schema: {{ schema_name }}
63+
schema: {{ schema }}
5664
file_format: delta
5765
threads: 4
66+
{%- endmacro %}
5867

59-
duckdb: &duckdb
68+
{%- macro duckdb_target(schema) %}
69+
duckdb:
6070
type: duckdb
6171
path: ":memory:"
62-
schema: {{ schema_name }}
72+
schema: {{ schema }}
6373
threads: 8
74+
{%- endmacro %}
6475

65-
sqlserver: &sqlserver
76+
{%- macro sqlserver_target(schema) %}
77+
sqlserver:
6678
type: sqlserver
6779
driver: "ODBC Driver 18 for SQL Server"
6880
server: 127.0.0.1
6981
port: 1433
7082
database: master
71-
schema: {{ schema_name }}
83+
schema: {{ schema }}
7284
user: sa
7385
password: "Elementary123!"
7486
encrypt: false
7587
trust_cert: true
7688
threads: 4
89+
{%- endmacro %}
7790

78-
vertica: &vertica
91+
{%- macro vertica_target(schema) %}
92+
vertica:
7993
type: vertica
8094
host: localhost
8195
port: 5433
8296
username: dbadmin
8397
password: vertica
8498
database: elementary_tests
85-
schema: {{ schema_name }}
99+
schema: {{ schema }}
86100
connection_load_balance: false
87101
retries: 2
88102
threads: 4
103+
{%- endmacro %}
89104

90-
# ── Cloud targets (secrets substituted at CI time) ─────────────────
91-
92-
snowflake: &snowflake
105+
{%- macro snowflake_target(schema) %}
106+
snowflake:
93107
type: snowflake
94108
account: {{ snowflake_account | toyaml }}
95109
user: {{ snowflake_user | toyaml }}
96110
password: {{ snowflake_password | toyaml }}
97111
role: {{ snowflake_role | toyaml }}
98112
database: {{ snowflake_database | toyaml }}
99113
warehouse: {{ snowflake_warehouse | toyaml }}
100-
schema: {{ schema_name }}
114+
schema: {{ schema }}
101115
threads: 4
116+
{%- endmacro %}
102117

103-
bigquery: &bigquery
118+
{%- macro bigquery_target(dataset) %}
119+
bigquery:
104120
type: bigquery
105121
method: service-account-json
106122
project: {{ bigquery_project | toyaml }}
107-
dataset: {{ schema_name }}
123+
dataset: {{ dataset }}
108124
keyfile_json: {{ bigquery_keyfile | toyaml }}
109125
location: US
110126
priority: interactive
111127
threads: 4
128+
{%- endmacro %}
112129

113-
redshift: &redshift
130+
{%- macro redshift_target(schema) %}
131+
redshift:
114132
type: redshift
115133
host: {{ redshift_host | toyaml }}
116134
user: {{ redshift_user | toyaml }}
117135
password: {{ redshift_password | toyaml }}
118136
port: {{ redshift_port | toyaml }}
119137
dbname: {{ redshift_dbname | toyaml }}
120-
schema: {{ schema_name }}
138+
schema: {{ schema }}
121139
threads: 4
140+
{%- endmacro %}
122141

123-
databricks_catalog: &databricks_catalog
142+
{%- macro databricks_catalog_target(schema) %}
143+
databricks_catalog:
124144
type: databricks
125145
host: {{ databricks_host | toyaml }}
126146
http_path: {{ databricks_http_path | toyaml }}
127147
catalog: {{ databricks_catalog | toyaml }}
128-
schema: {{ schema_name }}
148+
schema: {{ schema }}
129149
auth_type: oauth
130150
client_id: {{ databricks_client_id | toyaml }}
131151
client_secret: {{ databricks_client_secret | toyaml }}
132152
threads: 4
153+
{%- endmacro %}
133154

134-
fabric: &fabric
155+
{%- macro fabric_target(schema) %}
156+
fabric:
135157
type: fabric
136158
driver: "ODBC Driver 18 for SQL Server"
137159
server: {{ fabric_server | toyaml }}
138160
port: 1433
139161
database: {{ fabric_database | toyaml }}
140-
schema: {{ schema_name }}
162+
schema: {{ schema }}
141163
authentication: ServicePrincipal
142164
tenant_id: {{ fabric_tenant_id | toyaml }}
143165
client_id: {{ fabric_client_id | toyaml }}
144166
client_secret: {{ fabric_client_secret | toyaml }}
145167
encrypt: true
146168
trust_cert: false
147169
threads: 4
170+
{%- endmacro %}
148171

149-
athena: &athena
172+
{%- macro athena_target(schema) %}
173+
athena:
150174
type: athena
151175
s3_staging_dir: {{ athena_s3_staging_dir | toyaml }}
152176
s3_data_dir: {{ athena_s3_data_dir | toyaml }}
153177
region_name: {{ athena_region | toyaml }}
154178
database: awsdatacatalog
155-
schema: {{ schema_name }}
179+
schema: {{ schema }}
156180
aws_access_key_id: {{ athena_aws_access_key_id | toyaml }}
157181
aws_secret_access_key: {{ athena_aws_secret_access_key | toyaml }}
158182
threads: 4
183+
{%- endmacro %}
184+
185+
{#- ── Macro that emits all targets for a given schema suffix ─────── -#}
186+
{%- macro all_targets(schema) %}
187+
{{ postgres_target(schema) }}
188+
{{ clickhouse_target(schema) }}
189+
{{ trino_target(schema) }}
190+
{{ dremio_target(schema) }}
191+
{{ spark_target(schema) }}
192+
{{ duckdb_target(schema) }}
193+
{{ sqlserver_target(schema) }}
194+
{{ vertica_target(schema) }}
195+
{{ snowflake_target(schema) }}
196+
{{ bigquery_target(schema) }}
197+
{{ redshift_target(schema) }}
198+
{{ databricks_catalog_target(schema) }}
199+
{{ fabric_target(schema) }}
200+
{{ athena_target(schema) }}
201+
{%- endmacro %}
202+
203+
elementary_tests:
204+
target: postgres
205+
outputs:
206+
{{ all_targets(schema_name) }}
159207

160-
# The root dbt_project.yml uses profile "elementary", so we alias the
208+
# The root dbt_project.yml uses profile "elementary", so we define the
161209
# same targets but override the schema to <base>_elementary.
162210
elementary:
163211
target: postgres
164212
outputs:
165-
{%- set targets = ['postgres', 'clickhouse', 'trino', 'dremio', 'spark', 'duckdb', 'sqlserver', 'vertica', 'snowflake', 'bigquery', 'redshift', 'databricks_catalog', 'athena', 'fabric'] %}
166-
{%- for t in targets %}
167-
{{ t }}:
168-
<<: *{{ t }}
169-
{{ 'dataset' if t == 'bigquery' else 'schema' }}: {{ schema_name }}_elementary
170-
{%- endfor %}
213+
{{ all_targets(schema_name ~ '_elementary') }}

0 commit comments

Comments
 (0)