Skip to content

Commit 9c888fa

Browse files
haritamarharitamar
committed
Merge remote-tracking branch 'origin/master' into feature/volume-threshold-test
2 parents 9bfc0b7 + a025161 commit 9c888fa

File tree

402 files changed

+15298
-10397
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

402 files changed

+15298
-10397
lines changed

‎.dbt-fusion-version‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎.github/workflows/assign-devin-prs.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
name: Auto-assign Devin PRs
2+
3+
on:
4+
pull_request:
5+
types: [opened]
6+
7+
jobs:
8+
assign:
9+
if: github.actor == 'devin-ai-integration[bot]'
10+
runs-on: ubuntu-latest
11+
permissions:
12+
pull-requests: write
13+
issues: write
14+
steps:
15+
- name: Extract and assign requesting user
16+
uses: actions/github-script@v7
17+
with:
18+
script: |
19+
const body = context.payload.pull_request.body || '';
20+
const match = body.match(/Requested by[:\s]*(?:@(\w[\w-]*)|[\w][\w\s]*\(@(\w[\w-]*)\))/);
21+
const user = match?.[1] || match?.[2];
22+
if (user) {
23+
await github.rest.issues.addAssignees({
24+
owner: context.repo.owner,
25+
repo: context.repo.repo,
26+
issue_number: context.issue.number,
27+
assignees: [user]
28+
});
29+
console.log(`Assigned PR #${context.issue.number} to @${user}`);
30+
} else {
31+
console.log('Could not determine requesting user from PR description');
32+
}

‎.github/workflows/cleanup-stale-schemas.yml‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
name: Cleanup stale CI schemas
2+
3+
on:
4+
schedule:
5+
# Daily at 03:00 UTC
6+
- cron: "0 3 * * *"
7+
workflow_dispatch:
8+
inputs:
9+
max-age-hours:
10+
type: string
11+
required: false
12+
default: "24"
13+
description: Drop schemas older than this many hours
14+
15+
env:
16+
TESTS_DIR: ${{ github.workspace }}/dbt-data-reliability/integration_tests
17+
18+
jobs:
19+
cleanup:
20+
runs-on: ubuntu-latest
21+
strategy:
22+
fail-fast: false
23+
matrix:
24+
warehouse-type:
25+
- snowflake
26+
- bigquery
27+
- redshift
28+
- databricks_catalog
29+
- athena
30+
steps:
31+
- name: Checkout dbt package
32+
uses: actions/checkout@v4
33+
with:
34+
path: dbt-data-reliability
35+
36+
- name: Setup Python
37+
uses: actions/setup-python@v6
38+
with:
39+
python-version: "3.10"
40+
cache: "pip"
41+
42+
- name: Install dbt
43+
run: >
44+
pip install
45+
"dbt-core"
46+
"dbt-${{ (matrix.warehouse-type == 'databricks_catalog' && 'databricks') || (matrix.warehouse-type == 'athena' && 'athena-community') || matrix.warehouse-type }}"
47+
48+
- name: Write dbt profiles
49+
env:
50+
CI_WAREHOUSE_SECRETS: ${{ secrets.CI_WAREHOUSE_SECRETS || '' }}
51+
run: |
52+
# The cleanup job doesn't create schemas, but generate_profiles.py
53+
# requires --schema-name. Use a dummy value.
54+
python "${{ github.workspace }}/dbt-data-reliability/integration_tests/profiles/generate_profiles.py" \
55+
--template "${{ github.workspace }}/dbt-data-reliability/integration_tests/profiles/profiles.yml.j2" \
56+
--output ~/.dbt/profiles.yml \
57+
--schema-name "cleanup_placeholder"
58+
59+
- name: Install dbt deps
60+
working-directory: ${{ env.TESTS_DIR }}/dbt_project
61+
run: dbt deps
62+
63+
- name: Symlink local elementary package
64+
run: ln -sfn ${{ github.workspace }}/dbt-data-reliability ${{ env.TESTS_DIR }}/dbt_project/dbt_packages/elementary
65+
66+
- name: Drop stale CI schemas
67+
working-directory: ${{ env.TESTS_DIR }}/dbt_project
68+
# Only dbt_ prefixed schemas are created in this repo's CI.
69+
# The elementary repo has its own workflow for py_ prefixed schemas.
70+
run: >
71+
dbt run-operation drop_stale_ci_schemas
72+
--args '{prefixes: ["dbt_"], max_age_hours: ${{ inputs.max-age-hours || '24' }}}'
73+
-t "${{ matrix.warehouse-type }}"

‎.github/workflows/run-precommit.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ jobs:
1313
- name: Set up Python
1414
uses: actions/setup-python@v4.3.0
1515
with:
16-
python-version: "3.8"
16+
python-version: "3.10"
1717

1818
- name: Install dev requirements
1919
run: pip install -r dev-requirements.txt

‎.github/workflows/test-all-warehouses.yml‎

Lines changed: 59 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,56 @@ on:
3535
required: false
3636

3737
jobs:
38+
# ── Local targets ─────────────────────────────────────────────────────
39+
# No secrets needed — run on pull_request (works for forks without approval).
40+
# Skipped on pull_request_target to avoid duplicate runs for internal PRs.
41+
# Includes Docker-based adapters (postgres, clickhouse, trino, dremio) and
42+
# fully in-process adapters (duckdb).
43+
test-local:
44+
if: github.event_name != 'pull_request_target'
45+
strategy:
46+
fail-fast: false
47+
matrix:
48+
dbt-version:
49+
${{ inputs.dbt-version && fromJSON(format('["{0}"]', inputs.dbt-version)) ||
50+
fromJSON('["latest_official", "latest_pre"]') }}
51+
warehouse-type:
52+
[
53+
postgres,
54+
clickhouse,
55+
trino,
56+
dremio,
57+
spark,
58+
duckdb,
59+
sqlserver,
60+
vertica,
61+
]
62+
exclude:
63+
# latest_pre is only tested on postgres
64+
- dbt-version: latest_pre
65+
warehouse-type: clickhouse
66+
- dbt-version: latest_pre
67+
warehouse-type: trino
68+
- dbt-version: latest_pre
69+
warehouse-type: dremio
70+
- dbt-version: latest_pre
71+
warehouse-type: spark
72+
- dbt-version: latest_pre
73+
warehouse-type: duckdb
74+
- dbt-version: latest_pre
75+
warehouse-type: sqlserver
76+
- dbt-version: latest_pre
77+
warehouse-type: vertica
78+
uses: ./.github/workflows/test-warehouse.yml
79+
with:
80+
warehouse-type: ${{ matrix.warehouse-type }}
81+
dbt-version: ${{ matrix.dbt-version }}
82+
elementary-ref: ${{ inputs.elementary-ref }}
83+
dbt-data-reliability-ref: ${{ inputs.dbt-data-reliability-ref || (github.event_name == 'pull_request' && github.event.pull_request.head.sha) || '' }}
84+
85+
# ── Cloud targets ─────────────────────────────────────────────────────
86+
# Require secrets — use fork check / approval gate for pull_request_target.
87+
3888
# Determine if this is a fork PR and skip if wrong trigger is used
3989
check-fork-status:
4090
runs-on: ubuntu-latest
@@ -74,7 +124,7 @@ jobs:
74124
- name: Approved
75125
run: echo "Fork PR approved for testing"
76126

77-
test:
127+
test-cloud:
78128
needs: [check-fork-status, approve-fork]
79129
if: |
80130
! cancelled() &&
@@ -88,26 +138,19 @@ jobs:
88138
${{ inputs.dbt-version && fromJSON(format('["{0}"]', inputs.dbt-version)) ||
89139
fromJSON('["latest_official"]') }}
90140
warehouse-type:
91-
[
92-
postgres,
93-
snowflake,
94-
bigquery,
95-
redshift,
96-
databricks_catalog,
97-
athena,
98-
trino,
99-
clickhouse,
100-
dremio,
101-
]
141+
[snowflake, bigquery, redshift, databricks_catalog, athena, fabric]
142+
# Fusion includes: always run fusion alongside the base version for
143+
# supported warehouses. When inputs.dbt-version is already 'fusion' the
144+
# matrix deduplicates automatically.
102145
include:
103-
- dbt-version: "${{ inputs.dbt-version || 'latest_pre' }}"
104-
warehouse-type: postgres
105146
- dbt-version: "${{ inputs.dbt-version || 'fusion' }}"
106147
warehouse-type: snowflake
107148
- dbt-version: "${{ inputs.dbt-version || 'fusion' }}"
108149
warehouse-type: bigquery
109-
- dbt-version: "${{ inputs.dbt-version || 'fusion' }}"
110-
warehouse-type: redshift
150+
# fusion/redshift temporarily disabled - Fusion sidecar can't resolve
151+
# source schemas via SVV_COLUMNS on Redshift (see dbt-labs/dbt-fusion issues)
152+
# - dbt-version: "${{ inputs.dbt-version || 'fusion' }}"
153+
# warehouse-type: redshift
111154
- dbt-version: "${{ inputs.dbt-version || 'fusion' }}"
112155
warehouse-type: databricks_catalog
113156
uses: ./.github/workflows/test-warehouse.yml
@@ -117,19 +160,3 @@ jobs:
117160
elementary-ref: ${{ inputs.elementary-ref }}
118161
dbt-data-reliability-ref: ${{ inputs.dbt-data-reliability-ref || ((github.event_name == 'pull_request_target' || github.event_name == 'pull_request') && github.event.pull_request.head.sha) || '' }}
119162
secrets: inherit
120-
121-
notify_failures:
122-
name: Notify Slack
123-
secrets: inherit
124-
needs: [test]
125-
if: |
126-
always() &&
127-
! contains(needs.test.result, 'success') &&
128-
! contains(needs.test.result, 'cancelled') &&
129-
contains(github.event_name, 'schedule') &&
130-
! cancelled()
131-
uses: elementary-data/elementary/.github/workflows/notify_slack.yml@master
132-
with:
133-
result: "failure"
134-
run_id: ${{ github.run_id }}
135-
workflow_name: ${{ github.workflow }}

0 commit comments

Comments
 (0)