ci(min-deps): promote min-deps to first-class CI alongside latest-deps (#1480)

sd-db · web-flow · commit b90fb0a54d34 · 2026-05-26T12:07:06.000+05:30
## Summary

Promotes min-deps to a first-class CI citizen alongside latest-deps so
lower-bound regressions surface pre-merge on every PR — including forks.
Shared dep cache now covers both resolutions; min-deps unit + build run
on every PR; min-deps integration runs nightly + on `/integration-test
min-deps`.

## Test plan

- [x] Warm-cache populates new combined-key shape and lowest-direct
wheel closure.
- [x] Cache restore wired correctly (branch-scope dispatch run hits
cache, JFrog skipped).
- [x] PR checks green on `main.yml` + `main-min-deps.yml`.
- [x] `/integration-test` still routes to `integration.yml`.

**Post-merge:** dispatch `warmDepsCache.yml` on `main` once to populate
main-scope cache immediately. Need to check `/integration-test min-deps`
routing
diff --git a/.github/actions/setup-python-deps/action.yml b/.github/actions/setup-python-deps/action.yml
@@ -19,8 +19,8 @@ runs:
           ~/.cache/uv
           ~/.cache/pip
           ~/.cache/pip-wheelhouse
-        key: python-deps-${{ hashFiles('uv.lock', 'pyproject.toml') }}-latest
-        restore-keys: python-deps-${{ hashFiles('uv.lock', 'pyproject.toml') }}-
+        key: python-deps-${{ hashFiles('uv.lock', 'requirements.lowest-direct.txt', 'pyproject.toml') }}-latest
+        restore-keys: python-deps-${{ hashFiles('uv.lock', 'requirements.lowest-direct.txt', 'pyproject.toml') }}-
 
     - name: Restore pre-commit cache
       uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
diff --git a/.github/workflows/integration-min-deps.yml b/.github/workflows/integration-min-deps.yml
@@ -1,15 +1,37 @@
-# Min-Deps Integration Tests (slow)
+# Integration Tests (Min-Deps) for dbt-databricks
 #
 # Sharded functional tests against the lowest-direct dependency resolution
 # committed in requirements.lowest-direct.txt. Mirrors integration.yml's
 # fan-out (skip-if-unchanged on schedule, prepare-shards via LPT historical
 # timing, sharded execution per profile, gather-shards verification) but
 # routes every pytest invocation through the `min-deps:` hatch env.
 #
-# Triggering: schedule only.
-
-name: Min-Deps Tests (slow)
+# Triggering:
+# 1. On a PR (internal OR fork): a maintainer comments `/integration-test min-deps`.
+#    The integration-trigger workflow validates the comment author and
+#    dispatches this workflow with the PR number. The run posts a result
+#    comment back to the PR when the matrix completes.
+# 2. Manually from the Actions tab (workflow_dispatch):
+#    - One PR (e.g. "100") OR comma-separated list ("100,200,300") in pr_numbers.
+#    - Or a git_ref for ad-hoc testing.
+# 3. Nightly on `main` at 19:30 UTC (2h before integration.yml's 21:30 nightly).
+#    The `prepare` job short-circuits if the current main SHA has already had a
+#    successful min-deps integration run, emitting an empty targets array so
+#    the matrix jobs skip cleanly.
+
+name: Integration Tests (Min-Deps)
 on:
+  workflow_dispatch:
+    inputs:
+      pr_numbers:
+        description: "PR number(s) to test — single PR or comma-separated for batch (e.g. '100' or '100,200,300')"
+        required: false
+        type: string
+      git_ref:
+        description: "Git ref (branch/tag/commit) to test — used only when pr_numbers is empty"
+        required: false
+        type: string
+
   schedule:
     - cron: "30 19 * * *"   # 19:30 UTC, 2h before integration.yml's nightly
 
@@ -18,7 +40,7 @@ permissions:
   contents: read
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.event.inputs.pr_numbers || github.event.inputs.git_ref || github.ref }}
   cancel-in-progress: true
 
 defaults:
@@ -43,23 +65,46 @@ jobs:
         id: parse
         shell: bash
         env:
+          EVENT_NAME: ${{ github.event_name }}
+          INPUT_PR_NUMBERS: ${{ github.event.inputs.pr_numbers }}
+          INPUT_GIT_REF: ${{ github.event.inputs.git_ref }}
           DEFAULT_REF: ${{ github.ref }}
           GH_TOKEN: ${{ github.token }}
         run: |
           set -euo pipefail
           entry() { printf '{"pr":"%s","ref":"%s"}' "$1" "$2"; }
           targets="["
-          # Nightly skip-if-unchanged: if this main SHA already has a green
-          # min-deps-slow run, emit empty targets so the matrix jobs skip.
-          already_tested=$(curl -sfS \
-            -H "Authorization: Bearer $GH_TOKEN" \
-            -H "Accept: application/vnd.github+json" \
-            "https://api.github.com/repos/$GITHUB_REPOSITORY/actions/workflows/min-deps-test-slow.yml/runs?branch=main&status=success&head_sha=$GITHUB_SHA" \
-            | jq -r '.total_count // 0')
-          if [[ "$already_tested" -gt 0 ]]; then
-            echo "Nightly skip: main @ $GITHUB_SHA already has $already_tested successful run(s)."
+          if [[ "$EVENT_NAME" == "schedule" ]]; then
+            # Nightly skip-if-unchanged: if this main SHA already has a green
+            # min-deps integration run, emit empty targets so the matrix jobs skip.
+            already_tested=$(curl -sfS \
+              -H "Authorization: Bearer $GH_TOKEN" \
+              -H "Accept: application/vnd.github+json" \
+              "https://api.github.com/repos/$GITHUB_REPOSITORY/actions/workflows/integration-min-deps.yml/runs?branch=main&status=success&head_sha=$GITHUB_SHA" \
+              | jq -r '.total_count // 0')
+            if [[ "$already_tested" -gt 0 ]]; then
+              echo "Nightly skip: main @ $GITHUB_SHA already has $already_tested successful run(s)."
+            else
+              targets+=$(entry "nightly" "$DEFAULT_REF")
+            fi
+          elif [[ -n "${INPUT_PR_NUMBERS//[[:space:]]/}" ]]; then
+            first=1
+            IFS=',' read -ra prs <<< "$INPUT_PR_NUMBERS"
+            for pr in "${prs[@]}"; do
+              pr_trimmed="${pr//[[:space:]]/}"
+              [[ -z "$pr_trimmed" ]] && continue
+              if [[ ! "$pr_trimmed" =~ ^[0-9]+$ ]]; then
+                echo "::error::Invalid PR number '$pr_trimmed' in pr_numbers='$INPUT_PR_NUMBERS' — expected digits, comma-separated."
+                exit 1
+              fi
+              [[ $first -eq 0 ]] && targets+=","
+              first=0
+              targets+=$(entry "$pr_trimmed" "refs/pull/$pr_trimmed/head")
+            done
+          elif [[ -n "${INPUT_GIT_REF//[[:space:]]/}" ]]; then
+            targets+=$(entry "manual" "$INPUT_GIT_REF")
           else
-            targets+=$(entry "nightly" "$DEFAULT_REF")
+            targets+=$(entry "manual" "$DEFAULT_REF")
           fi
           targets+="]"
           echo "targets=$targets" >> "$GITHUB_OUTPUT"
@@ -97,7 +142,12 @@ jobs:
         with:
           ref: ${{ matrix.target.ref }}
 
-      - name: Setup JFrog PyPI Proxy
+      - name: Setup Python Dependencies
+        id: deps
+        uses: ./.github/actions/setup-python-deps
+
+      - name: Setup JFrog PyPI Proxy (fallback)
+        if: steps.deps.outputs.cache-hit != 'true'
         uses: ./.github/actions/setup-jfrog-pypi
 
       - name: Set up python
@@ -193,7 +243,12 @@ jobs:
         with:
           ref: ${{ matrix.target.ref }}
 
-      - name: Setup JFrog PyPI Proxy
+      - name: Setup Python Dependencies
+        id: deps
+        uses: ./.github/actions/setup-python-deps
+
+      - name: Setup JFrog PyPI Proxy (fallback)
+        if: steps.deps.outputs.cache-hit != 'true'
         uses: ./.github/actions/setup-jfrog-pypi
 
       - name: Set up python
@@ -287,7 +342,12 @@ jobs:
         with:
           ref: ${{ matrix.target.ref }}
 
-      - name: Setup JFrog PyPI Proxy
+      - name: Setup Python Dependencies
+        id: deps
+        uses: ./.github/actions/setup-python-deps
+
+      - name: Setup JFrog PyPI Proxy (fallback)
+        if: steps.deps.outputs.cache-hit != 'true'
         uses: ./.github/actions/setup-jfrog-pypi
 
       - name: Set up python
@@ -379,7 +439,12 @@ jobs:
         with:
           ref: ${{ matrix.target.ref }}
 
-      - name: Setup JFrog PyPI Proxy
+      - name: Setup Python Dependencies
+        id: deps
+        uses: ./.github/actions/setup-python-deps
+
+      - name: Setup JFrog PyPI Proxy (fallback)
+        if: steps.deps.outputs.cache-hit != 'true'
         uses: ./.github/actions/setup-jfrog-pypi
 
       - name: Set up python
@@ -511,3 +576,54 @@ jobs:
             echo "::error::One or more shard-verification invariants failed."
             exit 1
           fi
+
+  # Posts a per-job pass/fail summary comment back to the PR when dispatched
+  # with a single PR number (the slash-command path). Skipped for batch
+  # dispatches and for schedule / git_ref runs.
+  report-status:
+    needs:
+      - run-uc-cluster-e2e-tests
+      - run-sqlwarehouse-e2e-tests
+      - run-cluster-e2e-tests
+      - gather-shards
+    if: |
+      always() &&
+      github.event_name == 'workflow_dispatch' &&
+      inputs.pr_numbers != '' &&
+      !contains(inputs.pr_numbers, ',')
+    runs-on:
+      group: databricks-protected-runner-group
+      labels: linux-ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Post result comment
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b  # v7
+        env:
+          PR_NUMBER: ${{ inputs.pr_numbers }}
+          UC_RESULT: ${{ needs.run-uc-cluster-e2e-tests.result }}
+          SQLW_RESULT: ${{ needs.run-sqlwarehouse-e2e-tests.result }}
+          CLUSTER_RESULT: ${{ needs.run-cluster-e2e-tests.result }}
+          GATHER_RESULT: ${{ needs.gather-shards.result }}
+        with:
+          script: |
+            const ICONS = { success: ':white_check_mark:', skipped: ':fast_forward:' };
+            const results = {
+              'UC cluster':          process.env.UC_RESULT,
+              'SQL warehouse':       process.env.SQLW_RESULT,
+              'All-purpose cluster': process.env.CLUSTER_RESULT,
+              'Shard coverage':      process.env.GATHER_RESULT,
+            };
+            const line = Object.entries(results)
+              .map(([name, r]) => `${name} ${ICONS[r] || ':x:'} ${r}`)
+              .join(' · ');
+            const runUrl =
+              `https://github.com/${context.repo.owner}/${context.repo.repo}` +
+              `/actions/runs/${context.runId}`;
+            const prNumber = process.env.PR_NUMBER.trim();
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: parseInt(prNumber, 10),
+              body: `Min-deps integration results for PR #${prNumber} — ${line}\n\n[Run details](${runUrl}).`,
+            });
diff --git a/.github/workflows/integration-trigger.yml b/.github/workflows/integration-trigger.yml
@@ -1,9 +1,15 @@
 # Integration Test Trigger
 #
-# Listens for `/integration-test` PR comments and dispatches the integration
-# test workflow against the PR head. Works uniformly for internal and forked
-# PRs because the dispatch runs in the main repo context (which has access to
-# the Databricks secrets).
+# Listens for `/integration-test` PR comments and dispatches the appropriate
+# integration test workflow against the PR head. Works uniformly for internal
+# and forked PRs because the dispatch runs in the main repo context (which
+# has access to the Databricks secrets).
+#
+# Routing on the first whitespace-separated arg after `/integration-test`:
+#   /integration-test                  → integration.yml          (latest deps)
+#   /integration-test min-deps         → integration-min-deps.yml (lowest-direct lock)
+#   /integration-test <anything else>  → integration.yml          (trailing text
+#                                        is free-form context, e.g. retry notes)
 #
 # Authorization: only users whose author_association is OWNER, MEMBER, or
 # COLLABORATOR can trigger a run. Anyone else gets a reply comment explaining
@@ -22,8 +28,7 @@ permissions:
 jobs:
   dispatch:
     # Exact command match, or the command followed by whitespace (so maintainers
-    # can add context like `/integration-test please retry the sqlw flake`) —
-    # never matches `/integration-test-foo`.
+    # can add context or modifiers) — never matches `/integration-test-foo`.
     if: |
       github.event.issue.pull_request &&
       (github.event.comment.body == '/integration-test' || startsWith(github.event.comment.body, '/integration-test ')) &&
@@ -47,22 +52,26 @@ jobs:
         uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b  # v7
         with:
           script: |
+            const rest = context.payload.comment.body.slice('/integration-test'.length).trim();
+            const isMinDeps = rest === 'min-deps' || rest.startsWith('min-deps ');
+            const workflowId = isMinDeps ? 'integration-min-deps.yml' : 'integration.yml';
+            const label = isMinDeps ? 'Min-deps integration tests' : 'Integration tests';
             const prNumber = String(context.payload.issue.number);
             await github.rest.actions.createWorkflowDispatch({
               owner: context.repo.owner,
               repo: context.repo.repo,
-              workflow_id: 'integration.yml',
+              workflow_id: workflowId,
               ref: 'main',
               inputs: { pr_numbers: prNumber },
             });
             const actionsUrl =
               `https://github.com/${context.repo.owner}/${context.repo.repo}` +
-              `/actions/workflows/integration.yml`;
+              `/actions/workflows/${workflowId}`;
             await github.rest.issues.createComment({
               owner: context.repo.owner,
               repo: context.repo.repo,
               issue_number: context.payload.issue.number,
-              body: `Integration tests dispatched for PR #${prNumber} by @${context.payload.comment.user.login}. ` +
+              body: `${label} dispatched for PR #${prNumber} by @${context.payload.comment.user.login}. ` +
                     `Track progress in the [Actions tab](${actionsUrl}).`,
             });
 
diff --git a/.github/workflows/main-min-deps.yml b/.github/workflows/main-min-deps.yml
@@ -1,12 +1,8 @@
 # Min-deps variant of main.yml's Tests and Code Checks workflow. Runs
 # unit tests + a build+verify+smoke-parse job against the committed
-# lowest-direct lock. Fires on every merge to main / *.latest (always)
-# and on internal PRs (early signal); fork PRs are skipped at the job
-# level because they cannot mint the JFrog OIDC token the protected
-# runner needs.
-#
-# Sharded functional coverage lives in the companion min-deps-test-slow.yml
-# on a nightly schedule.
+# lowest-direct lock. Fires on every push to main / *.latest and on
+# every PR (including forks) — dependencies are served from the
+# pre-populated cache (see warmDepsCache.yml) when available.
 
 name: Tests and Code Checks (Min-Deps)
 
@@ -15,26 +11,35 @@ on:
     branches:
       - "main"
       - "*.latest"
+      - "releases/*"
+    paths-ignore:
+      - "**.MD"
+      - "**.md"
   pull_request:
+    paths-ignore:
+      - "**.MD"
+      - "**.md"
   workflow_dispatch:
 
 permissions:
   id-token: write
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}
+  cancel-in-progress: true
+
 defaults:
   run:
     shell: bash
 
 jobs:
   unit:
     name: unit test / python 3.10
-    # Fork PRs cannot mint the JFrog OIDC token; skip them on pull_request.
-    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
     runs-on:
       group: databricks-protected-runner-group
       labels: linux-ubuntu-latest
-    timeout-minutes: 25
+    timeout-minutes: 15
 
     env:
       UV_FROZEN: "1"
@@ -43,7 +48,12 @@ jobs:
       - name: Check out the repository
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
 
-      - name: Setup JFrog PyPI Proxy
+      - name: Setup Python Dependencies
+        id: deps
+        uses: ./.github/actions/setup-python-deps
+
+      - name: Setup JFrog PyPI Proxy (fallback)
+        if: steps.deps.outputs.cache-hit != 'true'
         uses: ./.github/actions/setup-jfrog-pypi
 
       - name: Set up Python
@@ -64,12 +74,9 @@ jobs:
 
   build:
     name: Build and Verify Packages
-    # Fork PRs cannot mint the JFrog OIDC token; skip them on pull_request.
-    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
     runs-on:
       group: databricks-protected-runner-group
       labels: linux-ubuntu-latest
-    timeout-minutes: 15
 
     env:
       UV_FROZEN: "1"
@@ -78,7 +85,12 @@ jobs:
       - name: Check out the repository
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4
 
-      - name: Setup JFrog PyPI Proxy
+      - name: Setup Python Dependencies
+        id: deps
+        uses: ./.github/actions/setup-python-deps
+
+      - name: Setup JFrog PyPI Proxy (fallback)
+        if: steps.deps.outputs.cache-hit != 'true'
         uses: ./.github/actions/setup-jfrog-pypi
 
       - name: Set up Python
diff --git a/.github/workflows/warmDepsCache.yml b/.github/workflows/warmDepsCache.yml
