Skip to content

Commit 9817afd

Browse files
devin-ai-integration[bot]devin-ai-integration[bot]
authored
refactor: replace opaque CI_PROFILES_YML secret with committed template + envsubst (#2123)
1 parent 866d7ae commit 9817afd

3 files changed

Lines changed: 223 additions & 3 deletions

File tree

‎.github/workflows/test-warehouse.yml‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -114,12 +114,15 @@ jobs:
114114
115115
- name: Write dbt profiles
116116
env:
117-
PROFILES_YML: ${{ secrets.CI_PROFILES_YML }}
117+
CI_WAREHOUSE_SECRETS: ${{ secrets.CI_WAREHOUSE_SECRETS || '' }}
118118
run: |
119-
mkdir -p ~/.dbt
120119
DBT_VERSION=$(pip show dbt-core | grep -i version | awk '{print $2}' | sed 's/\.//g')
121120
UNDERSCORED_REF_NAME=$(echo "${{ inputs.warehouse-type }}_dbt_${DBT_VERSION}_${BRANCH_NAME}" | awk '{print tolower($0)}' | head -c 40 | sed "s/[-\/]/_/g")
122-
echo "$PROFILES_YML" | base64 -d | sed "s/<SCHEMA_NAME>/py_$UNDERSCORED_REF_NAME/g" > ~/.dbt/profiles.yml
121+
122+
python "${{ github.workspace }}/elementary/tests/profiles/generate_profiles.py" \
123+
--template "${{ github.workspace }}/elementary/tests/profiles/profiles.yml.j2" \
124+
--output ~/.dbt/profiles.yml \
125+
--schema-name "py_$UNDERSCORED_REF_NAME"
123126
124127
- name: Run Python package unit tests
125128
run: pytest -vv tests/unit --warehouse-type ${{ inputs.warehouse-type }}

‎tests/profiles/generate_profiles.py‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
#!/usr/bin/env python3
2+
"""Generate ~/.dbt/profiles.yml from a Jinja2 template and an optional secrets JSON."""
3+
4+
from __future__ import annotations
5+
6+
import base64
7+
import binascii
8+
import json
9+
import os
10+
from pathlib import Path
11+
from typing import Any
12+
13+
import click
14+
import yaml
15+
from jinja2 import BaseLoader, Environment, StrictUndefined, Undefined
16+
17+
18+
class _NullUndefined(Undefined):
19+
"""Render missing variables as empty strings so docker-only runs don't crash."""
20+
21+
def __str__(self) -> str:
22+
return ""
23+
24+
def __iter__(self):
25+
return iter([])
26+
27+
def __bool__(self) -> bool:
28+
return False
29+
30+
31+
def _yaml_inline(value: Any) -> str:
32+
"""Dump *value* as a compact inline YAML scalar / mapping."""
33+
if isinstance(value, Undefined):
34+
return "{}"
35+
return yaml.dump(value, default_flow_style=True).strip()
36+
37+
38+
@click.command()
39+
@click.option(
40+
"--template",
41+
required=True,
42+
type=click.Path(exists=True, dir_okay=False, path_type=Path),
43+
help="Path to the Jinja2 profiles template (e.g. profiles.yml.j2).",
44+
)
45+
@click.option(
46+
"--output",
47+
required=True,
48+
type=click.Path(dir_okay=False, path_type=Path),
49+
help="Destination path for the rendered profiles.yml.",
50+
)
51+
@click.option(
52+
"--schema-name",
53+
required=True,
54+
help="Base schema name (e.g. dbt_pkg_<ref> or py_<ref>).",
55+
)
56+
@click.option(
57+
"--secrets-json-env",
58+
default="CI_WAREHOUSE_SECRETS",
59+
show_default=True,
60+
help="Name of the env-var holding the base64-encoded JSON secrets blob.",
61+
)
62+
def main(
63+
template: Path,
64+
output: Path,
65+
schema_name: str,
66+
secrets_json_env: str,
67+
) -> None:
68+
"""Render a Jinja2 profiles template into a dbt profiles.yml file.
69+
70+
Resolution order:
71+
1. If the env-var named by ``--secrets-json-env`` is set, decode it and
72+
use its key/value pairs (plus *schema_name*) as template variables.
73+
2. Otherwise render the template with only *schema_name* populated (all
74+
other variables resolve to empty strings — suitable for docker-only
75+
targets on fork PRs).
76+
"""
77+
output.parent.mkdir(parents=True, exist_ok=True)
78+
79+
secrets_b64 = os.environ.get(secrets_json_env, "").strip()
80+
81+
# ── Build template context ──────────────────────────────────────────
82+
context: dict[str, object] = {"schema_name": schema_name}
83+
84+
if secrets_b64:
85+
try:
86+
decoded: dict = json.loads(base64.b64decode(secrets_b64))
87+
except (binascii.Error, json.JSONDecodeError) as e:
88+
raise click.ClickException(
89+
f"Failed to decode ${secrets_json_env}: {e}"
90+
) from e
91+
if not isinstance(decoded, dict):
92+
raise click.ClickException(
93+
f"Expected JSON object for ${secrets_json_env}, "
94+
f"got {type(decoded).__name__}"
95+
)
96+
for key, value in decoded.items():
97+
context[key.lower()] = value
98+
click.echo(
99+
f"Loaded {len(decoded)} secret(s) from ${secrets_json_env}.",
100+
err=True,
101+
)
102+
else:
103+
click.echo(
104+
"No secrets found — rendering template for docker-only targets.",
105+
err=True,
106+
)
107+
108+
# ── Render ──────────────────────────────────────────────────────────
109+
# When secrets are loaded, use StrictUndefined so typos in secret keys
110+
# fail fast. For docker-only runs (no secrets) use _NullUndefined so
111+
# cloud placeholders silently resolve to empty strings.
112+
undefined_cls = StrictUndefined if secrets_b64 else _NullUndefined
113+
env = Environment(
114+
loader=BaseLoader(),
115+
undefined=undefined_cls,
116+
keep_trailing_newline=True,
117+
)
118+
env.filters["toyaml"] = _yaml_inline
119+
tmpl = env.from_string(template.read_text())
120+
rendered = tmpl.render(**context)
121+
output.write_text(rendered)
122+
click.echo(f"Wrote {output}", err=True)
123+
124+
125+
if __name__ == "__main__":
126+
main()

‎tests/profiles/profiles.yml.j2‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
elementary_tests:
2+
target: postgres
3+
outputs:
4+
5+
# ── Docker targets (plaintext, no secrets needed) ──────────────────
6+
7+
postgres: &postgres
8+
type: postgres
9+
host: 127.0.0.1
10+
port: 5432
11+
user: admin
12+
password: admin
13+
dbname: postgres
14+
schema: {{ schema_name }}
15+
threads: 32
16+
17+
clickhouse: &clickhouse
18+
type: clickhouse
19+
host: localhost
20+
port: 8123
21+
user: default
22+
password: default
23+
schema: {{ schema_name }}
24+
threads: 4
25+
26+
# ── Cloud targets (secrets substituted at CI time) ─────────────────
27+
28+
snowflake: &snowflake
29+
type: snowflake
30+
account: {{ snowflake_account }}
31+
user: {{ snowflake_user }}
32+
password: {{ snowflake_password }}
33+
role: {{ snowflake_role }}
34+
database: {{ snowflake_database }}
35+
warehouse: {{ snowflake_warehouse }}
36+
schema: {{ schema_name }}
37+
threads: 4
38+
39+
bigquery: &bigquery
40+
type: bigquery
41+
method: service-account-json
42+
project: {{ bigquery_project }}
43+
dataset: {{ schema_name }}
44+
keyfile_json: {{ bigquery_keyfile | toyaml }}
45+
location: US
46+
priority: interactive
47+
threads: 4
48+
49+
redshift: &redshift
50+
type: redshift
51+
host: {{ redshift_host }}
52+
user: {{ redshift_user }}
53+
password: {{ redshift_password }}
54+
port: {{ redshift_port }}
55+
dbname: {{ redshift_dbname }}
56+
schema: {{ schema_name }}
57+
threads: 4
58+
59+
databricks_catalog: &databricks_catalog
60+
type: databricks
61+
host: {{ databricks_host }}
62+
http_path: {{ databricks_http_path }}
63+
catalog: {{ databricks_catalog }}
64+
schema: {{ schema_name }}
65+
auth_type: oauth
66+
client_id: {{ databricks_client_id }}
67+
client_secret: {{ databricks_client_secret }}
68+
threads: 4
69+
70+
athena: &athena
71+
type: athena
72+
s3_staging_dir: {{ athena_s3_staging_dir }}
73+
s3_data_dir: {{ athena_s3_data_dir }}
74+
region_name: {{ athena_region }}
75+
database: awsdatacatalog
76+
schema: {{ schema_name }}
77+
aws_access_key_id: {{ athena_aws_access_key_id }}
78+
aws_secret_access_key: {{ athena_aws_secret_access_key }}
79+
threads: 4
80+
81+
# The internal CLI dbt_project uses profile "elementary", so we alias the
82+
# same targets but override the schema to <base>_elementary.
83+
elementary:
84+
target: postgres
85+
outputs:
86+
{%- set targets = ['postgres', 'clickhouse', 'snowflake', 'bigquery', 'redshift', 'databricks_catalog', 'athena'] %}
87+
{%- for t in targets %}
88+
{{ t }}:
89+
<<: *{{ t }}
90+
{{ 'dataset' if t == 'bigquery' else 'schema' }}: {{ schema_name }}_elementary
91+
{%- endfor %}

0 commit comments

Comments
 (0)