Skip to content

Commit 5447b19

Browse files
feat: add weekly cleanup workflow for stale CI schemas
Re-uses the elementary.drop_stale_ci_schemas macro from dbt-data-reliability (checked out at workflow time) to drop py_-prefixed CI schemas older than 24 hours from cloud warehouses. Runs weekly on Sunday 03:00 UTC. Co-Authored-By: Itamar Hartstein <haritamar@gmail.com>
1 parent 02174cb commit 5447b19

1 file changed

Lines changed: 74 additions & 0 deletions

File tree

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
name: Cleanup stale CI schemas
2+
3+
on:
4+
schedule:
5+
# Every Sunday at 03:00 UTC
6+
- cron: "0 3 * * 0"
7+
workflow_dispatch:
8+
inputs:
9+
max-age-hours:
10+
type: string
11+
required: false
12+
default: "24"
13+
description: Drop schemas older than this many hours
14+
15+
env:
16+
# Re-use the dbt-data-reliability integration-test project so we get the
17+
# cleanup macro (elementary.drop_stale_ci_schemas) without duplicating it.
18+
TESTS_DIR: ${{ github.workspace }}/dbt-data-reliability/integration_tests
19+
20+
jobs:
21+
cleanup:
22+
runs-on: ubuntu-latest
23+
strategy:
24+
fail-fast: false
25+
matrix:
26+
warehouse-type:
27+
- snowflake
28+
- bigquery
29+
- redshift
30+
- databricks_catalog
31+
- athena
32+
steps:
33+
- name: Checkout dbt package
34+
uses: actions/checkout@v4
35+
with:
36+
repository: elementary-data/dbt-data-reliability
37+
path: dbt-data-reliability
38+
39+
- name: Setup Python
40+
uses: actions/setup-python@v6
41+
with:
42+
python-version: "3.10"
43+
cache: "pip"
44+
45+
- name: Install dbt
46+
run: >
47+
pip install
48+
"dbt-core"
49+
"dbt-${{ (matrix.warehouse-type == 'databricks_catalog' && 'databricks') || (matrix.warehouse-type == 'athena' && 'athena-community') || matrix.warehouse-type }}"
50+
51+
- name: Write dbt profiles
52+
env:
53+
CI_WAREHOUSE_SECRETS: ${{ secrets.CI_WAREHOUSE_SECRETS || '' }}
54+
run: |
55+
# The cleanup job doesn't create schemas, but generate_profiles.py
56+
# requires --schema-name. Use a dummy value.
57+
python "${{ github.workspace }}/dbt-data-reliability/integration_tests/profiles/generate_profiles.py" \
58+
--template "${{ github.workspace }}/dbt-data-reliability/integration_tests/profiles/profiles.yml.j2" \
59+
--output ~/.dbt/profiles.yml \
60+
--schema-name "cleanup_placeholder"
61+
62+
- name: Install dbt deps
63+
working-directory: ${{ env.TESTS_DIR }}/dbt_project
64+
run: dbt deps
65+
66+
- name: Symlink local elementary package
67+
run: ln -sfn ${{ github.workspace }}/dbt-data-reliability ${{ env.TESTS_DIR }}/dbt_project/dbt_packages/elementary
68+
69+
- name: Drop stale CI schemas
70+
working-directory: ${{ env.TESTS_DIR }}/dbt_project
71+
run: >
72+
dbt run-operation elementary.drop_stale_ci_schemas
73+
--args '{prefixes: ["py_"], max_age_hours: ${{ inputs.max-age-hours || '24' }}}'
74+
-t "${{ matrix.warehouse-type }}"

0 commit comments

Comments
 (0)