Split internal vs fork builds (#9729)

Internal builds (from within the repo) will go through a straight
`pull_request` flow, which triggers faster and allows for testing
non-dbt changes like updates to uv.lock, etc.

This flow has access to secrets, so cannot be ran from a fork. If a
fork tries to edit the workflow file to run it, it will fail to get
secrets and just not run 🤷

The downside here is that internal PRs show "duplicate" statuses for
PRs:
- The `dbt-ci/...` ones for status checking (mandatory checks). In the
  case of a fork PR, they are generated and posted from outside the PR
  flow, so we need to keep them this way
- The `dbt CI internal / ...` ones are automatically created from the PR
  flow here, and post their status back to `dbt-ci/...` but cannot be
  hidden

Author: Pluies

.github/workflows/dbt_full_run.yml

@@ -1,30 +1,85 @@
-name: dbt CI full run
+name: dbt CI internal
 
-# Manually-triggered CI run across all dbt projects, without filtering on changes.
-# Useful to validate changes to shared tools (scripts, uv.lock, workflows) that the PR flow can't
-# exercise pre-merge.
+# CI for PRs from duneanalytics/spellbook branches. Runs the code of the PR branch directly
+# (including workflows, scripts and uv.lock), and needs secrets.
+# PRs from forks are handled by the two-part flow instead (dbt_pr_trigger.yml + dbt_pr_run.yml)
 #
-# This runs the code of the selected branch directly, so it will only work on branches from the
-# duneanalytics/spellbook repo. It will not work from forks.
+# Both flows report the same dbt-ci/<project> commit statuses for every project on every PR
+# ("skipped" when unaffected), so branch protection can require those regardless of where the
+# PR comes from and without deadlocking.
 
 on:
-  workflow_dispatch:
+  pull_request:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
+  plan:
+    # Internal PRs only: fork PRs don't get secrets here and go through the
+    # "Receive dbt PR" flow instead
+    if: github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+      statuses: write
+    outputs:
+      projects: ${{ steps.projects.outputs.projects }}
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Determine modified dbt projects
+        id: projects
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh api "repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" \
+            --paginate --jq '.[].filename' > changed_files.txt
+
+          projects=$(ls -d dbt_subprojects/*/ | cut -d/ -f2)
+          modified=""
+          # Changes to shared code or CI machinery affect every project
+          if grep -qE '^(dbt_macros/shared/|sources/|scripts/|\.github/workflows/|uv\.lock|pyproject\.toml)' changed_files.txt; then
+            modified="$projects"
+          else
+            for p in $projects; do
+              if grep -q "^dbt_subprojects/$p/" changed_files.txt; then
+                modified="$modified $p"
+              fi
+            done
+          fi
+          echo "projects=$(echo "$modified" | tr ' ' '\n' | jq -Rnc '[inputs | select(length > 0)]')" >> "$GITHUB_OUTPUT"
+
+      - name: Report initial statuses to PR
+        env:
+          GH_TOKEN: ${{ github.token }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          PROJECTS: ${{ steps.projects.outputs.projects }}
+        run: |
+          # Pending for modified projects; success for the rest so that
+          # per-project required checks don't block PRs that never run them.
+          for p in $(ls -d dbt_subprojects/*/ | cut -d/ -f2); do
+            if echo "$PROJECTS" | jq -e --arg p "$p" 'index($p)' > /dev/null; then
+              state=pending; description="dbt CI queued"
+            else
+              state=success; description="Skipped - no changes"
+            fi
+            gh api "repos/${{ github.repository }}/statuses/$HEAD_SHA" \
+              -f "state=$state" \
+              -f "context=dbt-ci/$p" \
+              -f "description=$description" \
+              -f "target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          done
+
   dbt-run:
+    needs: plan
+    if: needs.plan.outputs.projects != '[]'
     strategy:
       fail-fast: false
       matrix:
-        project:
-          - daily_spellbook
-          - dex
-          - hourly_spellbook
-          - solana
-          - tokens
+        project: ${{ fromJSON(needs.plan.outputs.projects) }}
     uses: ./.github/workflows/dbt_run.yml
     secrets: inherit
     permissions:
@@ -33,3 +88,5 @@ jobs:
       statuses: write
     with:
       project: ${{ matrix.project }}
+      pr_number: ${{ github.event.pull_request.number }}
+      head_sha: ${{ github.event.pull_request.head.sha }}

.github/workflows/dbt_pr_run.yml

@@ -1,4 +1,4 @@
-name: dbt CI
+name: dbt CI from fork
 
 # Second half of the fork-safe CI pipeline. This workflow has access to secrets.
 # It downloads the artefacts generated in "Receive dbt PR", and runs dbt CI from these.
@@ -18,9 +18,12 @@ concurrency:
 
 jobs:
   plan:
+    # head_repository check: this flow is for fork PRs only; in-repo PRs are
+    # built directly by dbt_full_run.yml
     if: >
       github.event.workflow_run.event == 'pull_request' &&
-      github.event.workflow_run.conclusion == 'success'
+      github.event.workflow_run.conclusion == 'success' &&
+      github.event.workflow_run.head_repository.full_name != github.repository
     runs-on: ubuntu-latest
     permissions:
       contents: read

.github/workflows/dbt_pr_trigger.yml

@@ -1,23 +1,21 @@
 name: Receive dbt PR
 
 # First half of the fork-safe CI pipeline. This workflow runs in the unprivileged pull_request
-# context and packages the dbt code + metadata as artefacts. The privileged "dbt CI" workflow
-# then picks them up via workflow_run and runs CI if needed.
+# context and packages the dbt code + metadata as artefacts. The privileged "dbt CI from fork"
+# workflow then picks them up via workflow_run and runs CI if needed.
 
 on:
   pull_request:
-    paths:
-      - dbt_subprojects/**
-      - sources/**
-      - dbt_macros/shared/**
-      - .github/workflows/**
 
 permissions:
   contents: read
   pull-requests: read
 
 jobs:
   build:
+    # Only run if this is a PR from a fork
+    # PRs from duneanalytics/spellbook run directly via dbt_full_run.yml instead
+    if: github.event.pull_request.head.repo.full_name != github.repository
     runs-on: ubuntu-latest
 
     steps: