ci.yml

name: CI

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
  # Merge queue: the queue builds each PR as speculatively merged onto the
  # current main and only lands it if this workflow is green on that result —
  # the race-free version of the "pull main and re-verify before merging"
  # discipline (AGENTS.md multi-agent §7/§10). Every workflow that produces a
  # branch-protection-required check MUST carry this trigger, or queue builds
  # wait forever on a check that never reports.
  merge_group:

# Superseded runs on the same PR/branch waste runners and delay feedback;
# cancel them. Push runs to main group by commit ref as well, so an in-flight
# main run is cancelled only by a newer main push.
concurrency:
  group: ci-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  filter:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: read
      pull-requests: read
    outputs:
      # On merge_group, everything counts as changed: dorny/paths-filter has no
      # merge_group support, and the queue build is the last validation before
      # main — the one place a skipped job can never be the right answer. A
      # skipped step's output is the empty string (falsy), so `|| 'true'`
      # supplies the merge-group value without touching PR/push behavior.
      docs: ${{ steps.changes.outputs.docs || 'true' }}
      core: ${{ steps.changes.outputs.core || 'true' }}
      console: ${{ steps.changes.outputs.console || 'true' }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v7

      - uses: dorny/paths-filter@v4
        id: changes
        if: github.event_name != 'merge_group'
        with:
          filters: |
            docs:
              - 'apps/docs/**'
              - 'content/**'
              - 'pnpm-lock.yaml'
              - '.github/workflows/ci.yml'
            core:
              - 'packages/**'
              - 'examples/**'
              - 'apps/!(docs)/**'
              - 'package.json'
              - 'pnpm-lock.yaml'
              - 'tsconfig.json'
              - '.github/workflows/ci.yml'
            # Build inputs of the vendored Console SPA — see the Console Pin Gate
            # job at the bottom of this file. `.objectui-sha` is a ROOT DOTFILE, so
            # it matches neither filter above: a pin-only diff skipped `core` and
            # `docs` alike, which is how #4288 moved the pin 76 commits with six of
            # fourteen checks skipped and nothing anywhere building the new SHA.
            # The last two entries are the "a change to the guard runs the guard"
            # rule the repo applies to every other filtered gate; they are close to
            # free here because an unmoved pin hits the dist cache.
            console:
              - '.objectui-sha'
              - 'scripts/build-console.sh'
              - 'scripts/check-console-sha.mjs'
              - '.github/workflows/ci.yml'

  test:
    # Sharded 2-way BY PACKAGE: a core-touching PR ran the affected suite
    # ~11½ min on one 4-vCPU runner — the longest pole in the whole workflow.
    # scripts/partition-test-shards.mjs splits the package list into two
    # deterministic, test-file-count-balanced halves (573/572 at the time of
    # writing) and each shard runs its half through turbo. NOT the dogfood
    # job's vitest --shard passthrough, deliberately: that works for dogfood
    # because it is ONE package with ~60 files, but applied workspace-wide,
    # vitest 4 hard-fails every package with fewer test files than the shard
    # count — and `--passWithNoTests` converts the failure into running NOTHING
    # on either shard (three packages have exactly one test file today). See
    # the script header for the verification.
    #
    # Branch protection requires the bare "Test Core" context, which a matrix
    # can never publish again — the test-gate job below carries that name
    # (the #3622 lesson; see dogfood-gate).
    name: Test Core (${{ matrix.shard }}/2)
    needs: filter
    if: needs.filter.outputs.core == 'true'
    runs-on: ubuntu-latest
    # Backstop only — the stall guard on the test steps is the primary
    # detector for a #4250-style hang and fires well before this. 30 min is
    # ~4× a normal sharded run (~6-7 min), with margin for a cold Turbo cache;
    # the old 45 left a hung job "running" for half an hour past any plausible
    # healthy finish.
    timeout-minutes: 30
    permissions:
      contents: read
    strategy:
      fail-fast: false
      matrix:
        shard: [1, 2]

    steps:
      - name: Checkout repository
        uses: actions/checkout@v7
        with:
          # Full history so `turbo --affected` can diff against the PR base.
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@v7
        with:
          node-version: '22'

      - name: Enable Corepack
        run: corepack enable

      - name: Verify pnpm version
        run: pnpm --version

      - name: Get pnpm store directory
        shell: bash
        run: |
          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV

      - name: Setup pnpm cache
        uses: actions/cache@v6
        with:
          path: ${{ env.STORE_PATH }}
          key: ${{ runner.os }}-pnpm-store-v3-${{ hashFiles('**/pnpm-lock.yaml') }}
          restore-keys: |
            ${{ runner.os }}-pnpm-store-v3-

      # Restore-only on PRs: PR-side saves (~5 turbo entries per push) churned
      # the repo's 10 GB Actions cache pool and evicted the main-branch seeds —
      # observed as sudden cold-cache spikes (Build Core 51s → 4m30s). PRs fall
      # back to main's entries via the prefix restore-keys; only main pushes
      # save (the "Save Turbo cache" step at the end of the job). Shard-scoped
      # key: each shard builds/tests a different half of the workspace (same
      # pattern as dogfood). Jobs that used to fall back to this job's
      # namespace fall back to Build Core now — neither single shard builds a
      # superset anymore.
      - name: Restore Turbo cache
        uses: actions/cache/restore@v6
        with:
          path: .turbo/cache
          key: ${{ runner.os }}-turbo-${{ github.job }}-${{ matrix.shard }}-${{ github.ref_name }}-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-turbo-${{ github.job }}-${{ matrix.shard }}-${{ github.ref_name }}-
            ${{ runner.os }}-turbo-${{ github.job }}-${{ matrix.shard }}-
            ${{ runner.os }}-turbo-${{ github.job }}-

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      # PRs: only test packages affected by the diff against the PR base.
      # spec sits at the root of the dependency graph, so spec-touching PRs
      # still run (close to) everything — but the many PRs that don't touch
      # spec skip the bulk of the 75-package matrix. Push to main and
      # merge-queue builds partition the FULL package list instead: the queue
      # result IS the next main, so it gets main's validation, not the PR's
      # affected-only subset. (Spec's suite runs here plain / uninstrumented;
      # the coverage-instrumented pass lives in the nightly coverage-nightly
      # workflow.)
      #
      # !@objectstack/dogfood: the ~7½-minute dogfood suite is the dedicated
      # Dogfood job's whole purpose, and both jobs run under the same `core`
      # filter — without the exclusion every core PR executed the suite twice
      # in parallel, and it dominated this job's critical path.
      #
      # `turbo ls` is experimental; the partition script asserts its output
      # shape loudly so an upgrade that changes it turns into a red step
      # naming the cause, not a silently empty shard. An EMPTY shard file must
      # short-circuit the test step below: `turbo run test` with zero --filter
      # args runs the entire workspace.
      - name: Compute this shard's package set
        env:
          TURBO_SCM_BASE: ${{ github.event.pull_request.base.sha }}
        run: |
          if [ "${{ github.event_name }}" = "pull_request" ]; then
            pnpm exec turbo ls --affected --output=json > "$RUNNER_TEMP/turbo-ls.json"
          else
            pnpm exec turbo ls --output=json > "$RUNNER_TEMP/turbo-ls.json"
          fi
          node scripts/partition-test-shards.mjs "$RUNNER_TEMP/turbo-ls.json" \
            --shard ${{ matrix.shard }}/2 --exclude @objectstack/dogfood \
            > "$RUNNER_TEMP/shard-packages.txt"
          echo "Packages on this shard:"
          cat "$RUNNER_TEMP/shard-packages.txt"

      # --concurrency=4: turbo's default (10) oversubscribes the 4-vCPU
      # hosted runner; matching the core count bounds peak memory and the
      # job is CPU-bound anyway.
      # run-with-stall-guard replaces the old `… 2>&1 | tee $log` +
      # `set -o pipefail` idiom: the guard tees combined output to the log
      # itself and propagates the suite's real exit status, so there is no
      # pipe whose status tee could mask (do not reintroduce `| tee`). Its
      # actual job is #4250: a run whose output freezes mid-suite while the
      # job sits in_progress. Silence past --stall-minutes is declared a
      # stall — a labeled red naming the last output line — instead of a
      # 20-minute wait for a human (or the job timeout) to notice. 10 min is
      # ~5× the longest healthy quiet gap.
      #
      # NODE_OPTIONS arms every node process (vitest workers included) to dump
      # a diagnostic report on SIGUSR2; on a stall the guard signals the frozen
      # process group and digests the reports into the log — exact JS stack for
      # a process whose event loop is alive, and a named "no report = blocked
      # loop" verdict for one that is sync-spinning. The next #4250 occurrence
      # identifies its own culprit instead of costing a diagnosis.
      - name: Run this shard's tests
        env:
          NODE_OPTIONS: --report-on-signal --report-signal=SIGUSR2 --report-directory=${{ runner.temp }}/stall-reports
        run: |
          if [ ! -s "$RUNNER_TEMP/shard-packages.txt" ]; then
            echo "No packages on this shard — nothing to test."
            exit 0
          fi
          FILTERS=$(sed 's/^/--filter=/' "$RUNNER_TEMP/shard-packages.txt" | tr '\n' ' ')
          mkdir -p "$RUNNER_TEMP/stall-reports"
          node scripts/run-with-stall-guard.mjs --log "$RUNNER_TEMP/test-core.log" --stall-minutes 10 \
            --report-dir "$RUNNER_TEMP/stall-reports" -- \
            pnpm turbo run test $FILTERS --concurrency=4

      # Runs even when the suite failed — that is when it earns its keep. A red
      # suite plus a GREEN completeness check means real test failures; a red
      # suite plus a RED completeness check means a worker died and the cases it
      # owned never ran, which reads almost identically in the log (#3812).
      - name: Test completeness guard
        if: always()
        run: |
          if [ ! -f "$RUNNER_TEMP/test-core.log" ]; then
            echo "No test log — the test step did not get far enough to produce one."
            exit 0
          fi
          node scripts/check-test-completeness.mjs "$RUNNER_TEMP/test-core.log"

      # A stall's full diagnostic reports (JS stacks, libuv handles, heap
      # summary per process) outlive the in-log digest — keep them so a #4250
      # occurrence can be dissected offline. Free when nothing stalled: the
      # directory is empty and if-no-files-found skips the upload. Shard-scoped
      # name so the two matrix jobs don't collide.
      - name: Upload stall diagnostic reports
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: stall-reports-test-core-${{ matrix.shard }}
          path: ${{ runner.temp }}/stall-reports/
          if-no-files-found: ignore
          retention-days: 14

      # Seed the shared Turbo cache from main only (see the restore step
      # above). always(): keep the seed fresh even when a test fails, matching
      # the old actions/cache post-step behavior. Shard-scoped key so the two
      # matrix jobs don't collide.
      - name: Save Turbo cache (main only)
        if: always() && github.event_name == 'push'
        uses: actions/cache/save@v6
        with:
          path: .turbo/cache
          key: ${{ runner.os }}-turbo-${{ github.job }}-${{ matrix.shard }}-${{ github.ref_name }}-${{ github.sha }}

  test-gate:
    # Stable required-check name for the sharded Test Core matrix — the exact
    # contract dogfood-gate documents below (#3622): branch protection requires
    # the bare "Test Core" context, and once the job is a matrix that context
    # can never appear again, deadlocking every PR. Keeping the contract HERE
    # means a future shard-count change cannot deadlock the repo. See
    # dogfood-gate for why `cancelled` passes and why this must not be
    # `if: !cancelled()` on the job.
    name: Test Core
    needs: test
    if: always()
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: read
    steps:
      - name: Verify test shard results
        run: |
          result="${{ needs.test.result }}"
          echo "test matrix aggregate result: $result"
          case "$result" in
            success|skipped|cancelled) echo "Test Core gate satisfied ($result)." ;;
            *) echo "::error::Test Core shards did not pass (aggregate result: $result)"; exit 1 ;;
          esac


  # ── Temporal conformance against live, non-UTC servers (ADR-0053 D-A3) ─────
  #
  # The datetime storage work (#3912/#3942) was verified against real servers
  # because every one of its bugs was invisible on all-UTC infrastructure: a
  # zone-naive write resolved in the SERVER's zone on Postgres, mysql2 rendered
  # a Date in the HOST's zone, and a bare YYYY-MM-DD comparand meant a
  # different midnight per dialect. The committed suites are opt-in
  # (OS_TEST_POSTGRES_URL / OS_TEST_MYSQL_URL) and skip without a server, so
  # without this job they would never run in CI and the seam could regress
  # silently — D-A3's exact concern.
  #
  # Every timezone here is deliberately DIFFERENT: servers at +08:00, the Node
  # process at America/New_York, assertions in UTC. Both suites assert they
  # are pointed at a non-UTC server, so a mis-provisioned service fails loudly
  # instead of letting the job pass vacuously — and each test step now asserts
  # the PROCESS zone too, which nothing did before: a dropped `TZ:` line
  # silently returned the whole job to UTC coverage with everything still
  # green.
  #
  # The job also carries the non-SQL half of that axis (core, formula,
  # driver-memory, driver-mongodb, service-analytics) — the other backends the
  # temporal conformance matrix holds to one standard. Its NAME still says
  # "live PG + MySQL" on purpose: the name IS the required check, so renaming
  # it would silently drop the gate wherever branch protection lists it — the
  # same trap the dogfood shards note below.
  temporal-conformance:
    name: Temporal Conformance (live PG + MySQL)
    needs: filter
    if: needs.filter.outputs.core == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 30
    permissions:
      contents: read

    services:
      postgres:
        image: postgres:16
        env:
          POSTGRES_PASSWORD: postgres
        ports:
          - 5432:5432
        options: >-
          --health-cmd="pg_isready -U postgres"
          --health-interval=5s
          --health-timeout=5s
          --health-retries=12
      mysql:
        # Real MySQL 8.0. The hands-on verification of #3942 ran on MariaDB
        # 10.11 — the stricter dialect for datetime literals — so this job is
        # the other half of the compatibility claim. `-h 127.0.0.1` forces the
        # ping over TCP: the image's init phase runs mysqld with networking
        # disabled, so a socket ping would report healthy before init finishes.
        image: mysql:8.0
        env:
          MYSQL_ROOT_PASSWORD: root
          MYSQL_DATABASE: conformance
        ports:
          - 3306:3306
        options: >-
          --health-cmd="mysqladmin ping -h 127.0.0.1 -uroot -proot"
          --health-interval=5s
          --health-timeout=5s
          --health-retries=24

    steps:
      - name: Checkout repository
        uses: actions/checkout@v7

      # Service containers cannot override the image command, so the non-UTC
      # zones are set post-start through each server's own mechanism. Echoed
      # back so a provisioning failure is visible in the log — though the
      # suites' own non-UTC guards are the real gate.
      - name: Point both servers at a non-UTC timezone
        run: |
          docker exec ${{ job.services.postgres.id }} psql -U postgres -c "ALTER SYSTEM SET timezone='Asia/Shanghai'"
          docker exec ${{ job.services.postgres.id }} psql -U postgres -c "SELECT pg_reload_conf()"
          docker exec ${{ job.services.postgres.id }} psql -U postgres -tAc "SHOW timezone"
          docker exec ${{ job.services.mysql.id }} mysql -uroot -proot -e "SET GLOBAL time_zone = '+08:00'"
          docker exec ${{ job.services.mysql.id }} mysql -uroot -proot -N -e "SELECT @@global.time_zone"

      - name: Setup Node.js
        uses: actions/setup-node@v7
        with:
          node-version: '22'

      - name: Enable Corepack
        run: corepack enable

      - name: Get pnpm store directory
        shell: bash
        run: |
          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV

      - name: Setup pnpm cache
        uses: actions/cache@v6
        with:
          path: ${{ env.STORE_PATH }}
          key: ${{ runner.os }}-pnpm-store-v3-${{ hashFiles('**/pnpm-lock.yaml') }}
          restore-keys: |
            ${{ runner.os }}-pnpm-store-v3-

      # Restore-only (same policy as every other job); falls back to the Build
      # Core namespace because that job builds every package (a superset of the
      # build closure this one needs) and its cache is seeded from main. It
      # used to fall back to Test Core, but that namespace is per-shard now and
      # neither single shard builds a superset.
      - name: Restore Turbo cache
        uses: actions/cache/restore@v6
        with:
          path: .turbo/cache
          key: ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-turbo-${{ github.job }}-
            ${{ runner.os }}-turbo-build-core-${{ github.ref_name }}-
            ${{ runner.os }}-turbo-build-core-

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      - name: Build driver-sql and its dependencies
        run: pnpm exec turbo run build --filter=@objectstack/driver-sql... --concurrency=4

      # The whole driver-sql suite runs under the skewed process zone — not
      # just the live-server files — so a TZ-sensitive assumption anywhere in
      # the driver's tests fails here before it can ship.
      #
      # run-with-stall-guard: same wiring as Test Core (see the comment there;
      # #4250). Both of this job's test legs have hit the same nondeterministic
      # stall (#4331) — once frozen mid-file in core's kernel.test, once silent
      # for 24 minutes AFTER every suite printed `Done` (a process that never
      # exited) — and each burned the full 30-minute job timeout to end as an
      # uninformative "The operation was canceled". The guard watches output
      # silence, so both shapes become a labeled red naming the last line after
      # 10 quiet minutes, and its process-group kill takes down whatever
      # refused to exit. `--log` is the guard's own tee (mandatory); no
      # completeness guard reads these files yet. The zone-assert `node -e`
      # stays outside the wrapper: a one-shot print cannot stall, and the
      # guard should time the suite only.
      - name: Run driver-sql suite against both live servers
        env:
          TZ: America/New_York
          OS_TEST_POSTGRES_URL: postgres://postgres:postgres@127.0.0.1:5432/postgres
          OS_TEST_MYSQL_URL: mysql://root:root@127.0.0.1:3306/conformance
        run: |
          # The axis is only real if the zone actually took. Without this, a
          # dropped `TZ:` line silently returns the job to UTC coverage and
          # everything still passes — the same vacuous-pass hole the live-server
          # suites close by asserting a non-UTC SERVER.
          node -e "const tz=Intl.DateTimeFormat().resolvedOptions().timeZone,off=new Date().getTimezoneOffset();if(!tz||tz==='UTC'||off===0){console.error('process zone is '+tz+' (offset '+off+') — this job must run skewed');process.exit(1)}console.log('process zone: '+tz+' (offset '+off+')')"
          node scripts/run-with-stall-guard.mjs --log "$RUNNER_TEMP/temporal-driver-sql.log" --stall-minutes 10 -- \
            pnpm --filter @objectstack/driver-sql test

      # The non-SQL half of the same axis. `driver-sql` has run under a skewed
      # process zone since #3979, but the other backends the temporal
      # conformance matrix holds to that standard never did — and they are the
      # ones whose correctness rests ENTIRELY on process-zone independence,
      # since they have no server to blame: `storageDatetimeValue` /
      # `storageTimeValue` fold an instant through UTC getters, the filter-token
      # resolver derives "today" from UTC calendar parts, and the analytics
      # bucketer does the same. Swap one `getUTC*` for its local twin and every
      # one of those silently shifts by the host's offset — the exact defect
      # ADR-0053 D-C1 records for a `Date` bound to a Postgres TIME column.
      #
      # Measured before adding this: all five suites already pass under
      # America/New_York (-5/-4, with DST), Asia/Kolkata (+05:30) and
      # Pacific/Chatham (+12:45). So this found nothing on the day it landed,
      # which is what a ratchet is for — it makes the property enforced rather
      # than incidental.
      - name: Build the non-SQL temporal backends
        run: >-
          pnpm exec turbo run build
          --filter=@objectstack/service-analytics...
          --filter=@objectstack/driver-memory...
          --filter=@objectstack/driver-mongodb...
          --filter=@objectstack/formula...
          --concurrency=4

      - name: Run the non-SQL temporal backends under the skewed process zone
        env:
          TZ: America/New_York
        run: |
          # The axis is only real if the zone actually took. Without this, a
          # dropped `TZ:` line silently returns the job to UTC coverage and
          # everything still passes — the same vacuous-pass hole the live-server
          # suites close by asserting a non-UTC SERVER.
          node -e "const tz=Intl.DateTimeFormat().resolvedOptions().timeZone,off=new Date().getTimezoneOffset();if(!tz||tz==='UTC'||off===0){console.error('process zone is '+tz+' (offset '+off+') — this job must run skewed');process.exit(1)}console.log('process zone: '+tz+' (offset '+off+')')"
          # Stall guard: same wiring and rationale as the driver-sql step above
          # (#4250/#4331) — the "silent after every suite printed Done"
          # occurrence was on THIS leg.
          node scripts/run-with-stall-guard.mjs --log "$RUNNER_TEMP/temporal-non-sql.log" --stall-minutes 10 -- \
            pnpm \
              --filter @objectstack/core \
              --filter @objectstack/formula \
              --filter @objectstack/driver-memory \
              --filter @objectstack/driver-mongodb \
              --filter @objectstack/service-analytics \
              test

  dogfood:
    # Sharded 2-way: the suite is ~60 independent test files, each booting its
    # own in-process app, and a single 4-vCPU runner needed ~7½ minutes for the
    # lot — the longest pole in the whole workflow. vitest partitions the file
    # list deterministically across shards; both shards must pass. If branch
    # protection lists "Dogfood Regression Gate" as a required check, it must be
    # updated to the two sharded check names.
    name: Dogfood Regression Gate (${{ matrix.shard }}/2)
    needs: filter
    if: needs.filter.outputs.core == 'true'
    runs-on: ubuntu-latest
    # Backstop only — the stall guard on the test step is the primary detector
    # for a #4250-style hang (see Test Core). 30 min is ~4× a shard (~7 min;
    # the verify-CLI pass that used to ride shard 1 is its own parallel job
    # now — dogfood-verify below).
    timeout-minutes: 30
    permissions:
      contents: read
    strategy:
      fail-fast: false
      matrix:
        shard: [1, 2]

    steps:
      - name: Checkout repository
        uses: actions/checkout@v7

      - name: Setup Node.js
        uses: actions/setup-node@v7
        with:
          node-version: '22'

      - name: Enable Corepack
        run: corepack enable

      - name: Verify pnpm version
        run: pnpm --version

      - name: Get pnpm store directory
        shell: bash
        run: |
          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV

      - name: Setup pnpm cache
        uses: actions/cache@v6
        with:
          path: ${{ env.STORE_PATH }}
          key: ${{ runner.os }}-pnpm-store-v3-${{ hashFiles('**/pnpm-lock.yaml') }}
          restore-keys: |
            ${{ runner.os }}-pnpm-store-v3-

      # Shard-scoped key: the turbo test hash differs per shard (pass-through
      # args are part of the task hash). Restore-only on PRs — see the Restore
      # Turbo cache comment in the test job; the save step at the end of this
      # job seeds from main only.
      - name: Restore Turbo cache
        uses: actions/cache/restore@v6
        with:
          path: .turbo/cache
          key: ${{ runner.os }}-turbo-${{ github.job }}-${{ matrix.shard }}-${{ github.ref_name }}-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-turbo-${{ github.job }}-${{ matrix.shard }}-${{ github.ref_name }}-
            ${{ runner.os }}-turbo-${{ github.job }}-${{ matrix.shard }}-

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      # Boots real example apps in-process (in-memory SQLite) and exercises them
      # through the real HTTP + service stack — catches runtime regressions that
      # build / unit tests / spec-liveness pass over (e.g. the #2018 tz-bucketing
      # break, which was green on every static gate). The `--` args reach the
      # package's `vitest run` and are hashed into the turbo task, so each
      # shard caches independently.
      # run-with-stall-guard: same wiring as Test Core (see the comment there;
      # #4250/#4314) — it tees to dogfood.log itself for the completeness guard
      # below, propagates the suite's real exit status (no `| tee` + pipefail),
      # and turns frozen output into a labeled red after 10 min of silence.
      # Dogfood boots real engines in-process — exactly the population #4250's
      # stalls came from. NODE_OPTIONS + --report-dir arm the stall forensics
      # (SIGUSR2 stack harvest) — see the Test Core comment.
      - name: Boot example apps and exercise real user flows
        env:
          NODE_OPTIONS: --report-on-signal --report-signal=SIGUSR2 --report-directory=${{ runner.temp }}/stall-reports
        run: |
          mkdir -p "$RUNNER_TEMP/stall-reports"
          node scripts/run-with-stall-guard.mjs --log "$RUNNER_TEMP/dogfood.log" --stall-minutes 10 \
            --report-dir "$RUNNER_TEMP/stall-reports" -- \
            pnpm turbo run test --filter=@objectstack/dogfood -- --shard=${{ matrix.shard }}/2

      # Dogfood boots real apps in-process, so a native/OOM abort is likelier
      # here than in the unit suites — and a shard that dies silently looks like
      # a shard that had less work.
      - name: Test completeness guard
        if: always()
        run: |
          if [ ! -f "$RUNNER_TEMP/dogfood.log" ]; then
            echo "No test log — the test step did not get far enough to produce one."
            exit 0
          fi
          node scripts/check-test-completeness.mjs "$RUNNER_TEMP/dogfood.log"

      # Same offline-forensics artifact as Test Core; shard-scoped name so the
      # two matrix jobs don't collide.
      - name: Upload stall diagnostic reports
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: stall-reports-dogfood-${{ matrix.shard }}
          path: ${{ runner.temp }}/stall-reports/
          if-no-files-found: ignore
          retention-days: 14

      # Seed the shared Turbo cache from main only (see the restore step
      # above); shard-scoped key so the two matrix jobs don't collide.
      - name: Save Turbo cache (main only)
        if: always() && github.event_name == 'push'
        uses: actions/cache/save@v6
        with:
          path: .turbo/cache
          key: ${{ runner.os }}-turbo-${{ github.job }}-${{ matrix.shard }}-${{ github.ref_name }}-${{ github.sha }}

  # Replaces the former auto-verify dogfood tests: runs the published
  # `objectstack verify` engine over each example app through the CLI —
  # auto-derived CRUD round-trip fidelity + the cross-owner RLS invariant.
  # Exits non-zero on a real runtime failure, so it gates like the tests did.
  #
  # Its own job, not a rider on dogfood shard 1: the ~4½-minute pass ran
  # SERIALLY after that shard's tests, making shard 1 (~12 min) nearly twice
  # shard 2 (~6½ min) — the second-longest pole in the workflow for no
  # parallelism reason. It reports through dogfood-gate (below) rather than
  # its own required context, so the branch-protection contract is unchanged.
  dogfood-verify:
    name: Dogfood Verify CLI
    needs: filter
    if: needs.filter.outputs.core == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 20
    permissions:
      contents: read

    steps:
      - name: Checkout repository
        uses: actions/checkout@v7

      - name: Setup Node.js
        uses: actions/setup-node@v7
        with:
          node-version: '22'

      - name: Enable Corepack
        run: corepack enable

      - name: Verify pnpm version
        run: pnpm --version

      - name: Get pnpm store directory
        shell: bash
        run: |
          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV

      - name: Setup pnpm cache
        uses: actions/cache@v6
        with:
          path: ${{ env.STORE_PATH }}
          key: ${{ runner.os }}-pnpm-store-v3-${{ hashFiles('**/pnpm-lock.yaml') }}
          restore-keys: |
            ${{ runner.os }}-pnpm-store-v3-

      # Restore-only, with no save step at all (the console-pin pattern): the
      # job's only build is the CLI closure, and the build-core fallbacks are
      # the entries that actually hit — that job builds a superset and is
      # seeded from main.
      - name: Restore Turbo cache
        uses: actions/cache/restore@v6
        with:
          path: .turbo/cache
          key: ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}-
            ${{ runner.os }}-turbo-${{ github.job }}-
            ${{ runner.os }}-turbo-build-core-${{ github.ref_name }}-
            ${{ runner.os }}-turbo-build-core-

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      # Build the CLI plus BOTH example apps' full dependency closures (the
      # `pkg...` filter form). When this pass rode dogfood shard 1 the whole
      # workspace was already built by the test step, so `--filter=@objectstack/cli`
      # alone sufficed; standalone, the bundled objectstack.config imports
      # runtime packages (e.g. @objectstack/connector-mcp for app-showcase)
      # whose dist nothing here had built — ERR_MODULE_NOT_FOUND at verify
      # time. The closure syntax keeps this self-maintaining as app deps move.
      - name: Verify example apps via the `objectstack verify` CLI
        run: |
          pnpm turbo run build \
            --filter=@objectstack/cli... \
            --filter=@objectstack/example-crm... \
            --filter=@objectstack/example-showcase... \
            --concurrency=4
          for app in examples/app-crm examples/app-showcase; do
            echo "::group::objectstack verify $app --rls"
            OS_LOG_LEVEL=error node packages/cli/bin/run.js verify --app "$app/objectstack.config.ts" --rls
            echo "::endgroup::"
          done

  dogfood-gate:
    # Stable required-check name for a SHARDED job (#3622 follow-up).
    #
    # Branch protection requires the context "Dogfood Regression Gate". Once
    # the job became a 2-way matrix its checks publish as "Dogfood Regression
    # Gate (1/2)" / "(2/2)" — the bare context could never appear again, so
    # EVERY pull request in the repo sat permanently BLOCKED (mergeable, all
    # checks green, merge button dead). #3622's own comment called for updating
    # branch protection; keeping the contract HERE instead means a future
    # shard-count change cannot deadlock the repo a second time.
    #
    # Also aggregates dogfood-verify (the CLI pass that used to ride shard 1),
    # so the one required context still covers everything it covered before
    # the split.
    #
    # `if: always()` + result inspection so a legitimately skipped matrix (the
    # `filter` job says no core paths changed) still satisfies the gate.
    name: Dogfood Regression Gate
    needs: [dogfood, dogfood-verify]
    if: always()
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: read
    steps:
      - name: Verify dogfood shard results
        run: |
          result="${{ needs.dogfood.result }}"
          echo "dogfood matrix aggregate result: $result"
          # cancelled is a run-lifecycle state, not a shard verdict (#3668):
          # with cancel-in-progress on, every superseded push cancelled the
          # in-flight dogfood matrix — the longest job, so almost always the
          # one still running — and the old `*)` fallthrough painted a false
          # red on the old SHA. Verified experimentally (run 30271824408, a
          # fail-fast matrix with one real failure + one cancelled sibling):
          # a real shard failure DOMINATES the aggregate — it reads "failure",
          # never "cancelled" — so "cancelled" here can only mean the whole
          # run was stopped from outside (supersession, or a manual cancel —
          # accepted trade-off), and passing it masks no regression.
          # Deliberately NOT `if: !cancelled()` on the job instead: a skipped
          # gate publishes no required-check context on the SHA, which is the
          # #3622 merge-deadlock all over again.
          verify_result="${{ needs['dogfood-verify'].result }}"
          echo "dogfood-verify result: $verify_result"
          fail=0
          for r in "dogfood:$result" "dogfood-verify:$verify_result"; do
            case "${r#*:}" in
              success|skipped|cancelled) echo "Gate leg ${r%%:*} satisfied (${r#*:})." ;;
              *) echo "::error::Gate leg ${r%%:*} did not pass (result: ${r#*:})"; fail=1 ;;
            esac
          done
          exit "$fail"

  build-core:
    name: Build Core
    needs: filter
    if: needs.filter.outputs.core == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 30
    permissions:
      contents: read

    steps:
      - name: Checkout repository
        uses: actions/checkout@v7

      - name: Setup Node.js
        uses: actions/setup-node@v7
        with:
          node-version: '22'

      - name: Enable Corepack
        run: corepack enable

      - name: Verify pnpm version
        run: pnpm --version

      - name: Get pnpm store directory
        shell: bash
        run: |
          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV

      - name: Setup pnpm cache
        uses: actions/cache@v6
        with:
          path: ${{ env.STORE_PATH }}
          key: ${{ runner.os }}-pnpm-store-v3-${{ hashFiles('**/pnpm-lock.yaml') }}
          restore-keys: |
            ${{ runner.os }}-pnpm-store-v3-

      # Restore-only on PRs: PR-side saves (~5 turbo entries per push) churned
      # the repo's 10 GB Actions cache pool and evicted the main-branch seeds —
      # observed as sudden cold-cache spikes (Build Core 51s → 4m30s). PRs fall
      # back to main's entries via the prefix restore-keys; only main pushes
      # save (the "Save Turbo cache" step at the end of the job).
      - name: Restore Turbo cache
        uses: actions/cache/restore@v6
        with:
          path: .turbo/cache
          key: ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}-
            ${{ runner.os }}-turbo-${{ github.job }}-

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      - name: Build packages (excluding docs)
        run: pnpm build

      # A package that builds with plain `tsc` and does not exclude tests emits
      # `dist/**/*.test.js`. Two costs, and the second is the dangerous one:
      #
      #   1. `files: ["dist"]` publishes the tests to npm.
      #   2. A package with no vitest config COLLECTS those compiled copies
      #      alongside the sources, so every `src/**/*.test.ts` also runs as a
      #      stale `dist/**/*.test.js` built at the last `pnpm build`. That
      #      silently defeats edits — a fix to a source test appears not to work
      #      because the run is still executing the pre-fix duplicate — and it
      #      lets a source test be edited to pass while its stale twin keeps
      #      asserting the old behaviour, with neither obviously wrong.
      #
      # `@objectstack/cli` shipped exactly that (81 test files / 849 tests where
      # its sources hold 58 / 581) until #4065 excluded them. The rest of the
      # repo builds with tsup, which emits only declared entry points — so this
      # gate exists to stop the NEXT tsc-built package repeating it, not to
      # re-check the one that was fixed.
      - name: No compiled test files in any dist
        run: |
          set -o pipefail
          found="$(find packages -type d -name node_modules -prune -o \
            -type f \( -name '*.test.js' -o -name '*.test.cjs' -o -name '*.test.mjs' \) \
            -path '*/dist/*' -print | sort)"
          if [ -n "$found" ]; then
            echo "::error::Compiled test files found in build output. A tsc-built package is"
            echo "::error::missing a test exclude in its tsconfig.build.json, so these ship to"
            echo "::error::npm AND run as stale duplicates of their own sources."
            echo "$found"
            exit 1
          fi
          echo "OK — no compiled test files in any package dist."

      - name: Verify build outputs
        run: |
          if [ ! -d "packages/spec/dist" ]; then
            echo "Error: packages/spec/dist directory not found"
            exit 1
          fi
          if [ ! -d "packages/spec/json-schema" ]; then
            echo "Error: packages/spec/json-schema directory not found"
            exit 1
          fi
          echo "Build outputs verified successfully"

      # Capability packages (services / triggers / plugins) are loaded by the
      # multi-tenant runtime via a DYNAMIC import of their published entry. If a
      # package ships a dts-only / half-built / 0-byte `dist` (an interrupted
      # build, or a package retired in source but still referenced), that import
      # resolves to nothing and the capability SILENTLY fails to load — e.g.
      # record-change automation never fires, with no user-visible signal. The
      # build config always emits JS, so this can only happen by accident; assert
      # every buildable capability package actually produced its declared runtime
      # entry. Self-maintaining: reads each package's own `main`, skips dirs with
      # no package.json (e.g. a retired service-feed/service-ai leftover).
      - name: Verify capability packages ship a runtime entry (no dts-only / half-built)
        run: |
          fail=0
          for d in packages/triggers/* packages/services/* packages/plugins/*; do
            [ -f "$d/package.json" ] || continue
            has_build=$(node -p "Boolean((require('./$d/package.json').scripts||{}).build)" 2>/dev/null || echo false)
            [ "$has_build" = "true" ] || continue
            main=$(node -p "require('./$d/package.json').main || 'dist/index.js'" 2>/dev/null || echo dist/index.js)
            if [ ! -s "$d/$main" ]; then
              echo "✗ $d: missing/empty runtime entry '$main' — dts-only or unbuilt? dynamic import would silently fail"
              fail=1
            fi
          done
          if [ "$fail" -ne 0 ]; then exit 1; fi
          echo "✓ all buildable capability packages ship a runtime JS entry"

      - name: Analyze bundle size
        run: pnpm --filter @objectstack/spec analyze

      - name: Upload build artifacts
        uses: actions/upload-artifact@v7
        with:
          name: build-output
          path: |
            packages/spec/dist/
            packages/spec/json-schema/
          retention-days: 30

      # Seed the shared Turbo cache from main only (see the restore step above).
      - name: Save Turbo cache (main only)
        if: always() && github.event_name == 'push'
        uses: actions/cache/save@v6
        with:
          path: .turbo/cache
          key: ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}

  build-docs:
    name: Build Docs
    needs: filter
    if: needs.filter.outputs.docs == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 30
    permissions:
      contents: read

    steps:
      - name: Checkout repository
        uses: actions/checkout@v7

      - name: Setup Node.js
        uses: actions/setup-node@v7
        with:
          node-version: '22'

      - name: Enable Corepack
        run: corepack enable

      - name: Verify pnpm version
        run: pnpm --version

      - name: Get pnpm store directory
        shell: bash
        run: |
          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV

      - name: Setup pnpm cache
        uses: actions/cache@v6
        with:
          path: ${{ env.STORE_PATH }}
          key: ${{ runner.os }}-pnpm-store-v3-${{ hashFiles('**/pnpm-lock.yaml') }}
          restore-keys: |
            ${{ runner.os }}-pnpm-store-v3-
      
      - name: Setup Next.js cache
        uses: actions/cache@v6
        with:
          path: apps/docs/.next/cache
          key: ${{ runner.os }}-nextjs-${{ hashFiles('**/pnpm-lock.yaml') }}-${{ hashFiles('apps/docs/**.[jt]s', 'apps/docs/**.[jt]sx', 'content/**/*.mdx') }}
          restore-keys: |
            ${{ runner.os }}-nextjs-${{ hashFiles('**/pnpm-lock.yaml') }}-

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      - name: Build Docs
        run: pnpm --filter @objectstack/docs build

  # ── The pinned objectui SHA actually builds (#4290) ───────────────────────
  #
  # `.objectui-sha` is the single source of truth for the vendored Console SPA:
  # release.yml reads it, clones objectstack-ai/objectui at that commit, builds
  # @object-ui/console, and copies the dist into packages/console/. Editing that
  # one line replaces the entire frontend the platform ships.
  #
  # Nothing on the PR side ever built it. The pin appeared in exactly two
  # workflows, neither of them a gate: release.yml (post-merge, on the main push)
  # and showcase-smoke.yml (manual + nightly, whose own header says "it never
  # gates PRs"). Combined with the filter hole noted above, #4288 merged a
  # 76-commit pin bump green with six of fourteen checks skipped. A SHA that
  # cannot build — a typo, a commit that was force-pushed away, a genuinely
  # broken objectui, a dead bundle canary — reached main unchallenged and would
  # have surfaced at PUBLISH time. The only thing standing between the two was
  # the author remembering to run scripts/build-console.sh locally: discipline,
  # not a gate.
  #
  # `pnpm objectui:refresh` does include the build, so the normal path was
  # already covered in practice; the hole is a hand-edited pin, or that single
  # line cherry-picked out of another branch.
  #
  # Cost is real but narrowly aimed. A moved pin is the only trigger that pays
  # the full clone + vite build, and #4288's 76 commits of staleness are the
  # measure of how rare that is. The other triggers (this file, the drift guard)
  # leave the pin untouched, so they hit the dist cache and cost about a minute.
  #
  # Scope, stated plainly: this proves the PINNED SHA builds. It does NOT cover a
  # packages/client change breaking the injected-client bundle — that input lives
  # under `packages/**`, and watching it here would rebuild the console on a large
  # fraction of every PR. release.yml stays the only check for that direction.
  #
  # The NAME is the required-check contract — the same trap the dogfood shards
  # note above: renaming it silently drops the gate wherever branch protection
  # lists it, with every PR still green.
  console-pin:
    name: Console Pin Gate
    needs: filter
    if: needs.filter.outputs.console == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 45
    permissions:
      contents: read

    steps:
      - name: Checkout repository
        uses: actions/checkout@v7

      - name: Setup Node.js
        uses: actions/setup-node@v7
        with:
          node-version: '22'

      - name: Enable Corepack
        run: corepack enable

      - name: Verify pnpm version
        run: pnpm --version

      - name: Get pnpm store directory
        shell: bash
        run: |
          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV

      - name: Setup pnpm cache
        uses: actions/cache@v6
        with:
          path: ${{ env.STORE_PATH }}
          key: ${{ runner.os }}-pnpm-store-v3-${{ hashFiles('**/pnpm-lock.yaml') }}
          restore-keys: |
            ${{ runner.os }}-pnpm-store-v3-

      # Restore-only, and with no save step at all: this job runs a handful of
      # times a month, so its own namespace would almost always be cold. The
      # build-core fallbacks below are the ones that actually hit — that job
      # builds a superset of the client closure needed here and is seeded from
      # main (same reasoning as the Temporal Conformance job's fallback).
      - name: Restore Turbo cache
        uses: actions/cache/restore@v6
        with:
          path: .turbo/cache
          key: ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}-
            ${{ runner.os }}-turbo-${{ github.job }}-
            ${{ runner.os }}-turbo-build-core-${{ github.ref_name }}-
            ${{ runner.os }}-turbo-build-core-

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      # build-console.sh bundles THIS tree's @objectstack/client into the console
      # (via OBJECTSTACK_CLIENT_DIST) and falls back to building it in place when
      # the dist is missing — but that fallback is a bare `pnpm build` inside
      # packages/client, which tsup cannot finish unless @objectstack/core and
      # @objectstack/spec are built already. Build the closure through turbo
      # instead: cacheable, and near-free off the restore above. Both of the
      # script's existing callers (release.yml, showcase-smoke.yml) pre-build for
      # the same reason.
      - name: Build @objectstack/client and its dependencies
        run: pnpm exec turbo run build --filter=@objectstack/client... --concurrency=4

      # Same key as release.yml's "Cache vendored Console dist" step. Actions
      # caches are repo-scoped, so a PR reads whatever main already built for this
      # pin — that is what makes the non-pin triggers cheap. Only a pin the repo
      # has never built misses, and a miss is exactly when this gate has work to do.
      #
      # Split restore/save where release.yml uses the combined action, because the
      # combined form's post-step saves even when the job FAILED. build-console.sh
      # writes the SHA stamp before it asserts the bundle canary, so a canary
      # failure leaves a stamped-but-broken dist on disk — cache that and the next
      # run restores it, skips the build, and sails through the stamp check. Saving
      # only once every assertion below is green is what lets a restored dist carry
      # the same guarantees as a freshly built one.
      - name: Restore vendored Console dist (keyed on the objectui pin)
        id: console-dist
        uses: actions/cache/restore@v6
        with:
          path: packages/console/dist
          key: ${{ runner.os }}-console-dist-${{ hashFiles('.objectui-sha', 'scripts/build-console.sh') }}

      # The gate. Shallow-clones objectui at the pinned SHA, builds
      # @object-ui/console against this tree's client, copies the dist into
      # packages/console/, and asserts the bundle canary — every failure mode a
      # bad pin can carry, surfaced on the PR instead of at publish time.
      - name: Build the Console SPA at the pinned objectui SHA
        if: steps.console-dist.outputs.cache-hit != 'true'
        run: bash scripts/build-console.sh

      # check:console-sha exits 0 — deliberately — when there is no dist at all
      # (published installs and package-only CI legitimately have none), and warns
      # without failing when a dist carries no stamp. Both are right for that
      # script and both are VACUOUS here, where producing a dist is the entire
      # point: an empty or partial cache restore would otherwise reach the drift
      # check with nothing to check and report success. Prove one is really in
      # place before letting that check speak for the job.
      - name: Assert a real Console dist is in place
        env:
          CACHE_HIT: ${{ steps.console-dist.outputs.cache-hit }}
        run: |
          dist=packages/console/dist
          if [ "$CACHE_HIT" = "true" ]; then origin="the dist cache restore"; else origin="scripts/build-console.sh"; fi
          fail=0
          [ -s "$dist/index.html" ] || { echo "::error::$dist/index.html is missing or empty"; fail=1; }
          [ -n "$(ls -A "$dist/assets" 2>/dev/null || true)" ] || { echo "::error::$dist/assets is missing or empty"; fail=1; }
          [ -s "$dist/.objectui-sha" ] || { echo "::error::$dist/.objectui-sha stamp is missing — check:console-sha would pass vacuously"; fail=1; }
          if [ "$fail" -ne 0 ]; then
            echo "::error::No usable Console dist after $origin."
            exit 1
          fi
          echo "✓ Console dist present ($(du -sk "$dist" | awk '{print $1}') KB, via $origin) from objectui@$(cut -c1-12 "$dist/.objectui-sha")"

      - name: Verify the Console dist stamp matches the pin
        run: pnpm check:console-sha

      # Reached only with every assertion above green (see the split-restore note).
      # This deviates from the workflow's restore-only-on-PRs policy on purpose:
      # that policy targets the ~5 turbo entries every push writes, whereas this
      # key moves only when the pin does. A pin-bump PR writes one entry, and the
      # rest of that PR's pushes reuse it instead of rebuilding the console again.
      - name: Save vendored Console dist
        if: steps.console-dist.outputs.cache-hit != 'true'
        uses: actions/cache/save@v6
        with:
          path: packages/console/dist
          key: ${{ runner.os }}-console-dist-${{ hashFiles('.objectui-sha', 'scripts/build-console.sh') }}