From 5447b195c08d5a1f9d12dc906b2594c72dd6834f Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 28 Feb 2026 11:31:32 +0000 Subject: [PATCH 1/5] feat: add weekly cleanup workflow for stale CI schemas Re-uses the elementary.drop_stale_ci_schemas macro from dbt-data-reliability (checked out at workflow time) to drop py_-prefixed CI schemas older than 24 hours from cloud warehouses. Runs weekly on Sunday 03:00 UTC. Co-Authored-By: Itamar Hartstein --- .github/workflows/cleanup-stale-schemas.yml | 74 +++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 .github/workflows/cleanup-stale-schemas.yml diff --git a/.github/workflows/cleanup-stale-schemas.yml b/.github/workflows/cleanup-stale-schemas.yml new file mode 100644 index 000000000..53895b5e5 --- /dev/null +++ b/.github/workflows/cleanup-stale-schemas.yml @@ -0,0 +1,74 @@ +name: Cleanup stale CI schemas + +on: + schedule: + # Every Sunday at 03:00 UTC + - cron: "0 3 * * 0" + workflow_dispatch: + inputs: + max-age-hours: + type: string + required: false + default: "24" + description: Drop schemas older than this many hours + +env: + # Re-use the dbt-data-reliability integration-test project so we get the + # cleanup macro (elementary.drop_stale_ci_schemas) without duplicating it. + TESTS_DIR: ${{ github.workspace }}/dbt-data-reliability/integration_tests + +jobs: + cleanup: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + warehouse-type: + - snowflake + - bigquery + - redshift + - databricks_catalog + - athena + steps: + - name: Checkout dbt package + uses: actions/checkout@v4 + with: + repository: elementary-data/dbt-data-reliability + path: dbt-data-reliability + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.10" + cache: "pip" + + - name: Install dbt + run: > + pip install + "dbt-core" + "dbt-${{ (matrix.warehouse-type == 'databricks_catalog' && 'databricks') || (matrix.warehouse-type == 'athena' && 'athena-community') || matrix.warehouse-type }}" + + - name: Write dbt profiles + env: + CI_WAREHOUSE_SECRETS: ${{ secrets.CI_WAREHOUSE_SECRETS || '' }} + run: | + # The cleanup job doesn't create schemas, but generate_profiles.py + # requires --schema-name. Use a dummy value. + python "${{ github.workspace }}/dbt-data-reliability/integration_tests/profiles/generate_profiles.py" \ + --template "${{ github.workspace }}/dbt-data-reliability/integration_tests/profiles/profiles.yml.j2" \ + --output ~/.dbt/profiles.yml \ + --schema-name "cleanup_placeholder" + + - name: Install dbt deps + working-directory: ${{ env.TESTS_DIR }}/dbt_project + run: dbt deps + + - name: Symlink local elementary package + run: ln -sfn ${{ github.workspace }}/dbt-data-reliability ${{ env.TESTS_DIR }}/dbt_project/dbt_packages/elementary + + - name: Drop stale CI schemas + working-directory: ${{ env.TESTS_DIR }}/dbt_project + run: > + dbt run-operation elementary.drop_stale_ci_schemas + --args '{prefixes: ["py_"], max_age_hours: ${{ inputs.max-age-hours || '24' }}}' + -t "${{ matrix.warehouse-type }}" From d70c86b1720c2593baee8e83107b8e895e5f318b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 28 Feb 2026 17:26:04 +0000 Subject: [PATCH 2/5] fix: update workflow to call macro without elementary. prefix The drop_stale_ci_schemas macro moved from the main elementary package to the integration_tests project in dbt-data-reliability. Co-Authored-By: Itamar Hartstein --- .github/workflows/cleanup-stale-schemas.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cleanup-stale-schemas.yml b/.github/workflows/cleanup-stale-schemas.yml index 53895b5e5..b36f19aa4 100644 --- a/.github/workflows/cleanup-stale-schemas.yml +++ b/.github/workflows/cleanup-stale-schemas.yml @@ -14,7 +14,7 @@ on: env: # Re-use the dbt-data-reliability integration-test project so we get the - # cleanup macro (elementary.drop_stale_ci_schemas) without duplicating it. + # cleanup macro (drop_stale_ci_schemas) without duplicating it. TESTS_DIR: ${{ github.workspace }}/dbt-data-reliability/integration_tests jobs: @@ -69,6 +69,6 @@ jobs: - name: Drop stale CI schemas working-directory: ${{ env.TESTS_DIR }}/dbt_project run: > - dbt run-operation elementary.drop_stale_ci_schemas + dbt run-operation drop_stale_ci_schemas --args '{prefixes: ["py_"], max_age_hours: ${{ inputs.max-age-hours || '24' }}}' -t "${{ matrix.warehouse-type }}" From 225a5fbd0e0c1022d1d18bcaa1bb54b93192906d Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 28 Feb 2026 20:19:35 +0000 Subject: [PATCH 3/5] chore: change cleanup schedule from weekly to daily Co-Authored-By: Itamar Hartstein --- .github/workflows/cleanup-stale-schemas.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cleanup-stale-schemas.yml b/.github/workflows/cleanup-stale-schemas.yml index b36f19aa4..38f298968 100644 --- a/.github/workflows/cleanup-stale-schemas.yml +++ b/.github/workflows/cleanup-stale-schemas.yml @@ -2,8 +2,8 @@ name: Cleanup stale CI schemas on: schedule: - # Every Sunday at 03:00 UTC - - cron: "0 3 * * 0" + # Daily at 03:00 UTC + - cron: "0 3 * * *" workflow_dispatch: inputs: max-age-hours: From c2080877f68162ec1a643ae55c0d52f5663a46c3 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 28 Feb 2026 20:25:50 +0000 Subject: [PATCH 4/5] fix: address CodeRabbit review comments - Pin dbt-core and adapter versions to >=1.8,<1.10 - Validate max-age-hours input is a non-negative integer - Fail fast when CI_WAREHOUSE_SECRETS secret is missing Co-Authored-By: Itamar Hartstein --- .github/workflows/cleanup-stale-schemas.yml | 23 +++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cleanup-stale-schemas.yml b/.github/workflows/cleanup-stale-schemas.yml index 38f298968..4203e5fc8 100644 --- a/.github/workflows/cleanup-stale-schemas.yml +++ b/.github/workflows/cleanup-stale-schemas.yml @@ -45,13 +45,17 @@ jobs: - name: Install dbt run: > pip install - "dbt-core" - "dbt-${{ (matrix.warehouse-type == 'databricks_catalog' && 'databricks') || (matrix.warehouse-type == 'athena' && 'athena-community') || matrix.warehouse-type }}" + "dbt-core>=1.8,<1.10" + "dbt-${{ (matrix.warehouse-type == 'databricks_catalog' && 'databricks') || (matrix.warehouse-type == 'athena' && 'athena-community') || matrix.warehouse-type }}>=1.8,<1.10" - name: Write dbt profiles env: CI_WAREHOUSE_SECRETS: ${{ secrets.CI_WAREHOUSE_SECRETS || '' }} run: | + if [ -z "$CI_WAREHOUSE_SECRETS" ]; then + echo "::error::Missing required secret: CI_WAREHOUSE_SECRETS" + exit 1 + fi # The cleanup job doesn't create schemas, but generate_profiles.py # requires --schema-name. Use a dummy value. python "${{ github.workspace }}/dbt-data-reliability/integration_tests/profiles/generate_profiles.py" \ @@ -68,7 +72,14 @@ jobs: - name: Drop stale CI schemas working-directory: ${{ env.TESTS_DIR }}/dbt_project - run: > - dbt run-operation drop_stale_ci_schemas - --args '{prefixes: ["py_"], max_age_hours: ${{ inputs.max-age-hours || '24' }}}' - -t "${{ matrix.warehouse-type }}" + env: + MAX_AGE_HOURS: ${{ inputs.max-age-hours || '24' }} + run: | + if ! [[ "$MAX_AGE_HOURS" =~ ^[0-9]+$ ]]; then + echo "::error::max-age-hours must be a non-negative integer" + exit 1 + fi + ARGS=$(printf '{"prefixes":["py_"],"max_age_hours":%s}' "$MAX_AGE_HOURS") + dbt run-operation drop_stale_ci_schemas \ + --args "$ARGS" \ + -t "${{ matrix.warehouse-type }}" From a4954271eea4ff81229e8bd96bab023901a77447 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 28 Feb 2026 20:36:53 +0000 Subject: [PATCH 5/5] fix: remove dbt version pin to match other workflows Co-Authored-By: Itamar Hartstein --- .github/workflows/cleanup-stale-schemas.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cleanup-stale-schemas.yml b/.github/workflows/cleanup-stale-schemas.yml index 4203e5fc8..96b228965 100644 --- a/.github/workflows/cleanup-stale-schemas.yml +++ b/.github/workflows/cleanup-stale-schemas.yml @@ -45,8 +45,8 @@ jobs: - name: Install dbt run: > pip install - "dbt-core>=1.8,<1.10" - "dbt-${{ (matrix.warehouse-type == 'databricks_catalog' && 'databricks') || (matrix.warehouse-type == 'athena' && 'athena-community') || matrix.warehouse-type }}>=1.8,<1.10" + "dbt-core" + "dbt-${{ (matrix.warehouse-type == 'databricks_catalog' && 'databricks') || (matrix.warehouse-type == 'athena' && 'athena-community') || matrix.warehouse-type }}" - name: Write dbt profiles env: