Skip to content

Commit ac63b89

Browse files
devin-ai-integration[bot]haritamar
devin-ai-integration[bot]
and
haritamar
committed
refactor: replace opaque CI_PROFILES_YML secret with committed template + envsubst
- Add profiles.yml.template with plaintext docker targets and ${VAR} cloud placeholders - Update test-warehouse.yml to use CI_WAREHOUSE_SECRETS with envsubst (+ PROFILES_YML fallback) - Split test-all-warehouses.yml into test-docker (pull_request) and test-cloud (pull_request_target) - Guard BigQuery keyfile extraction against missing key - Use explicit envsubst '$SCHEMA_NAME' for fork PR path Co-Authored-By: Itamar Hartstein <haritamar@gmail.com>
1 parent 866d7ae commit ac63b89

3 files changed

Lines changed: 153 additions & 11 deletions

File tree

‎.github/workflows/test-all-warehouses.yml‎

Lines changed: 23 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,27 @@ on:
3535
description: Whether to generate new data
3636

3737
jobs:
38+
# ── Docker targets ────────────────────────────────────────────────────
39+
# No secrets needed — run on pull_request (works for forks without approval).
40+
# Skipped on pull_request_target to avoid duplicate runs for internal PRs.
41+
test-docker:
42+
if: github.event_name != 'pull_request_target'
43+
strategy:
44+
fail-fast: false
45+
matrix:
46+
dbt-version: ${{ inputs.dbt-version && fromJSON(format('["{0}"]', inputs.dbt-version)) || fromJSON('[null]') }}
47+
warehouse-type: [postgres, clickhouse]
48+
uses: ./.github/workflows/test-warehouse.yml
49+
with:
50+
warehouse-type: ${{ matrix.warehouse-type }}
51+
elementary-ref: ${{ inputs.elementary-ref || ((github.event_name == 'pull_request_target' || github.event_name == 'pull_request') && github.event.pull_request.head.sha) || '' }}
52+
dbt-data-reliability-ref: ${{ inputs.dbt-data-reliability-ref }}
53+
dbt-version: ${{ matrix.dbt-version }}
54+
generate-data: ${{ inputs.generate-data || false }}
55+
56+
# ── Cloud targets ─────────────────────────────────────────────────────
57+
# Require secrets — use fork check / approval gate for pull_request_target.
58+
3859
# Determine if this is a fork PR and skip if wrong trigger is used
3960
check-fork-status:
4061
runs-on: ubuntu-latest
@@ -74,7 +95,7 @@ jobs:
7495
- name: Approved
7596
run: echo "Fork PR approved for testing"
7697

77-
test:
98+
test-cloud:
7899
needs: [check-fork-status, approve-fork]
79100
if: |
80101
! cancelled() &&
@@ -86,15 +107,7 @@ jobs:
86107
matrix:
87108
dbt-version: ${{ inputs.dbt-version && fromJSON(format('["{0}"]', inputs.dbt-version)) || fromJSON('[null]') }}
88109
warehouse-type:
89-
[
90-
postgres,
91-
snowflake,
92-
bigquery,
93-
redshift,
94-
databricks_catalog,
95-
athena,
96-
clickhouse,
97-
]
110+
[snowflake, bigquery, redshift, databricks_catalog, athena]
98111
uses: ./.github/workflows/test-warehouse.yml
99112
with:
100113
warehouse-type: ${{ matrix.warehouse-type }}

‎.github/workflows/test-warehouse.yml‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,12 +114,51 @@ jobs:
114114
115115
- name: Write dbt profiles
116116
env:
117+
# New lean JSON secret (base64 encoded)
118+
SECRETS_JSON: ${{ secrets.CI_WAREHOUSE_SECRETS || '' }}
119+
# Backwards-compat fallback while migrating secrets
117120
PROFILES_YML: ${{ secrets.CI_PROFILES_YML }}
118121
run: |
119122
mkdir -p ~/.dbt
123+
124+
if ! command -v envsubst >/dev/null 2>&1; then
125+
sudo apt-get update
126+
sudo apt-get install -y gettext-base
127+
fi
128+
120129
DBT_VERSION=$(pip show dbt-core | grep -i version | awk '{print $2}' | sed 's/\.//g')
121130
UNDERSCORED_REF_NAME=$(echo "${{ inputs.warehouse-type }}_dbt_${DBT_VERSION}_${BRANCH_NAME}" | awk '{print tolower($0)}' | head -c 40 | sed "s/[-\/]/_/g")
122-
echo "$PROFILES_YML" | base64 -d | sed "s/<SCHEMA_NAME>/py_$UNDERSCORED_REF_NAME/g" > ~/.dbt/profiles.yml
131+
export SCHEMA_NAME="py_$UNDERSCORED_REF_NAME"
132+
133+
if [ -n "$SECRETS_JSON" ]; then
134+
DECODED=$(echo "$SECRETS_JSON" | base64 -d)
135+
BQ_KEYFILE_CONTENT=$(echo "$DECODED" | jq -r '.BIGQUERY_KEYFILE // empty')
136+
if [ -n "$BQ_KEYFILE_CONTENT" ]; then
137+
BIGQUERY_KEYFILE_PATH="$(mktemp /tmp/bigquery_keyfile.XXXXXX.json)"
138+
echo "$BQ_KEYFILE_CONTENT" > "$BIGQUERY_KEYFILE_PATH"
139+
chmod 600 "$BIGQUERY_KEYFILE_PATH"
140+
export BIGQUERY_KEYFILE_PATH
141+
fi
142+
143+
while IFS= read -r entry; do
144+
key=$(jq -r '.key' <<<"$entry")
145+
value=$(jq -r '.value' <<<"$entry")
146+
if [[ "$key" =~ ^[A-Z_][A-Z0-9_]*$ ]]; then
147+
printf -v "$key" '%s' "$value"
148+
export "$key"
149+
else
150+
echo "Skipping invalid secret key: $key" >&2
151+
fi
152+
done < <(echo "$DECODED" | jq -c 'to_entries[] | select(.key != "BIGQUERY_KEYFILE" and (.value | type == "string"))')
153+
154+
envsubst < "${{ github.workspace }}/elementary/tests/profiles/profiles.yml.template" > ~/.dbt/profiles.yml
155+
elif [ -n "$PROFILES_YML" ]; then
156+
echo "$PROFILES_YML" | base64 -d | sed "s/<SCHEMA_NAME>/$SCHEMA_NAME/g" > ~/.dbt/profiles.yml
157+
else
158+
# Fork PRs: no secrets available — only substitute SCHEMA_NAME so
159+
# cloud-target placeholders stay as-is (only docker targets work).
160+
envsubst '$SCHEMA_NAME' < "${{ github.workspace }}/elementary/tests/profiles/profiles.yml.template" > ~/.dbt/profiles.yml
161+
fi
123162
124163
- name: Run Python package unit tests
125164
run: pytest -vv tests/unit --warehouse-type ${{ inputs.warehouse-type }}

‎tests/profiles/profiles.yml.template‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
elementary_tests:
2+
target: postgres
3+
outputs: &outputs
4+
5+
# ── Docker targets (plaintext, no secrets needed) ──────────────────
6+
7+
postgres:
8+
type: postgres
9+
host: 127.0.0.1
10+
port: 5432
11+
user: admin
12+
password: admin
13+
dbname: postgres
14+
schema: ${SCHEMA_NAME}
15+
threads: 32
16+
17+
clickhouse:
18+
type: clickhouse
19+
host: localhost
20+
port: 8123
21+
user: default
22+
password: default
23+
schema: ${SCHEMA_NAME}
24+
threads: 4
25+
26+
# ── Cloud targets (secrets substituted at CI time) ─────────────────
27+
28+
snowflake:
29+
type: snowflake
30+
account: ${SNOWFLAKE_ACCOUNT}
31+
user: ${SNOWFLAKE_USER}
32+
password: ${SNOWFLAKE_PASSWORD}
33+
role: ${SNOWFLAKE_ROLE}
34+
database: ${SNOWFLAKE_DATABASE}
35+
warehouse: ${SNOWFLAKE_WAREHOUSE}
36+
schema: ${SCHEMA_NAME}
37+
threads: 4
38+
39+
bigquery:
40+
type: bigquery
41+
method: service-account
42+
project: ${BIGQUERY_PROJECT}
43+
dataset: ${SCHEMA_NAME}
44+
keyfile: ${BIGQUERY_KEYFILE_PATH}
45+
threads: 4
46+
47+
redshift:
48+
type: redshift
49+
host: ${REDSHIFT_HOST}
50+
user: ${REDSHIFT_USER}
51+
password: ${REDSHIFT_PASSWORD}
52+
port: ${REDSHIFT_PORT}
53+
dbname: ${REDSHIFT_DBNAME}
54+
schema: ${SCHEMA_NAME}
55+
threads: 4
56+
57+
databricks_catalog:
58+
type: databricks
59+
host: ${DATABRICKS_HOST}
60+
http_path: ${DATABRICKS_HTTP_PATH}
61+
catalog: ${DATABRICKS_CATALOG}
62+
schema: ${SCHEMA_NAME}
63+
client_id: ${DATABRICKS_CLIENT_ID}
64+
client_secret: ${DATABRICKS_CLIENT_SECRET}
65+
threads: 4
66+
67+
spark:
68+
type: spark
69+
method: http
70+
host: ${SPARK_HOST}
71+
token: ${SPARK_TOKEN}
72+
cluster: ${SPARK_CLUSTER}
73+
schema: ${SCHEMA_NAME}
74+
threads: 4
75+
76+
athena:
77+
type: athena
78+
s3_staging_dir: ${ATHENA_S3_STAGING_DIR}
79+
s3_data_dir: ${ATHENA_S3_DATA_DIR}
80+
region_name: ${ATHENA_REGION}
81+
database: awsdatacatalog
82+
schema: ${SCHEMA_NAME}
83+
aws_access_key_id: ${ATHENA_AWS_ACCESS_KEY_ID}
84+
aws_secret_access_key: ${ATHENA_AWS_SECRET_ACCESS_KEY}
85+
threads: 4
86+
87+
# The internal CLI dbt_project uses profile "elementary", so we alias the same outputs.
88+
elementary:
89+
target: postgres
90+
outputs: *outputs

0 commit comments

Comments
 (0)