nightly.yml

# ----------------------------------------------------------------------
# .github/workflows/nightly.yml
#    linkdb full coverage CI (Tier 2 of 3-tier CI model).
#
#    Runs the heavy parts that fast.yml intentionally skips, sharded for
#    wallclock:
#      - linux-pg-regress: PG 219 three-mode (own job, off the TAP path)
#      - linux-cluster-tap: cluster_tap files split into 9 parallel shards
#        (foundation-core / foundation-stage1 / stage2-ic-cssd / stage2-gcs /
#        stage3-core / stage3-recent / stage4-wal / stage4-hardgate /
#        stage5-ges-locking), with Stage 2/3 acceptance files kept in their
#        dedicated medium jobs.  cluster_unit + cluster_regress run once on the
#        foundation-core shard.
#      - build-test-macos: PG 219 + cluster_unit + cluster_regress + a curated
#        cluster_tap SMOKE subset (the full cluster_tap on macOS was the
#        wallclock bottleneck; Linux is the primary target)
#
#    Per CLAUDE.md rule 20.A revised, ship-time tag verification requires
#    both fast.yml green (push gate) AND nightly.yml green (this workflow)
#    on the same commit (or its main-branch ancestor).
#
# Author: SqlRush <sqlrush@gmail.com>
#
# Portions Copyright (c) 2026, pgrac contributors
#
# Triggers:
#   - schedule cron 08:00 UTC daily (off-peak for both US/EU/AP)
#   - workflow_dispatch (manual trigger by maintainer)
#   - push to tags v* (mandatory ship-time verification)
#
# Total wallclock target: ≤ 7 min (Linux TAP shards + pg-regress + macOS
# smoke all run in parallel; the slowest shard is the bound).  Was ~26 min
# (single macOS full cluster_tap job) before sharding + macOS smoke, then
# ~8-9 min with 3 broad Linux TAP shards.
# Trade-off: more runner-minutes (each shard self-builds) for less wallclock,
# per the maintainer's "GitHub job sharding is more stable than prove -j"
# guidance (TAP has port/tmp/IPC/cluster-node resource contention).
# ----------------------------------------------------------------------

name: linkdb nightly full CI

env:
  CCACHE_DIR: ${{ github.workspace }}/.ccache

on:
  schedule:
    - cron: '0 8 * * *'
  workflow_dispatch:
  push:
    tags: ['v*']

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: false

jobs:

  # ====================================================================
  # Tier 2.1a — Linux PG 219 three-mode regression (own job)
  # --------------------------------------------------------------------
  # Kept off the cluster_tap shard critical path: PG 219 (~5 min) on any
  # TAP shard would dominate it.  Build + install + make check only.
  # ====================================================================
  linux-pg-regress:
    name: Linux PG 219 regression
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install Linux dependencies
        run: |
          sudo apt-get update -y
          sudo apt-get install -y --no-install-recommends \
            build-essential ccache \
            libreadline-dev zlib1g-dev libicu-dev \
            liblz4-dev libzstd-dev libssl-dev libxml2-dev \
            libipc-run-perl pkg-config

      - name: Restore ccache
        uses: actions/cache@v4
        with:
          path: .ccache
          key: ${{ runner.os }}-${{ github.job }}-ccache-${{ github.ref_name }}
          restore-keys: |
            ${{ runner.os }}-${{ github.job }}-ccache-
            ${{ runner.os }}-ccache-

      - name: Configure ccache
        run: |
          ccache --max-size=700M
          ccache --zero-stats

      - name: Configure (--enable-cluster)
        run: |
          CC="ccache gcc" ./configure \
            --prefix=$HOME/pgrac-install \
            --enable-cassert --enable-debug \
            --with-openssl --with-icu --with-lz4 --with-zstd \
            --enable-cluster --enable-tap-tests

      - name: Build and install
        run: |
          make -j$(getconf _NPROCESSORS_ONLN)
          ccache --show-stats
          make install

      - name: Run PG regression tests (219 tests)
        run: make check

      - name: Upload regression diffs on failure
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: nightly-pg-regress-diffs
          path: |
            src/test/regress/regression.diffs
            src/test/regress/regression.out
            src/test/regress/log/
          if-no-files-found: ignore
          retention-days: 14

  # ====================================================================
  # Tier 2.1b — Linux cluster_tap (9 parallel shards)
  # --------------------------------------------------------------------
  # Shards by leading test-file number.  Stage acceptance files are not
  # duplicated here:
  #   - 200-202 are owned by stage2-acceptance-medium below.
  #   - 226-227 are owned by stage3-acceptance-medium below.
  #
  # cluster_unit + cluster_regress run only in foundation-core; the other
  # shards run their TAP slice only.  This saves repeated unit/regress work
  # and keeps the longest Linux TAP shard below the macOS smoke bound.
  # ====================================================================
  linux-cluster-tap:
    name: Linux cluster_tap (${{ matrix.shard.name }})
    runs-on: ubuntu-latest
    timeout-minutes: 45
    strategy:
      fail-fast: false
      matrix:
        shard:
          - { name: foundation-core,   ranges: "001-030", unit: true,  regress: true }
          - { name: foundation-stage1, ranges: "050-071", unit: false, regress: false }
          - { name: stage2-ic-cssd,    ranges: "072-099", unit: false, regress: false }
          - { name: stage2-gcs,        ranges: "100-118", unit: false, regress: false }
          - { name: stage3-core,       ranges: "203-225", unit: false, regress: false }
          - { name: stage3-recent,     ranges: "228-237", unit: false, regress: false }
          - { name: stage4-wal,        ranges: "242-248 273 275", unit: false, regress: false }
          - { name: stage4-hardgate,   ranges: "274-274", unit: false, regress: false }
          - { name: stage5-ges-locking, ranges: "276-326", unit: false, regress: false }
          # spec-5.19 D9: Stage 5 integrated acceptance (heavy 4-node ClusterQuad
          # + pgbench + crash-recovery) gets its own shard so the multi-node
          # legs run in parallel and do not extend the ges-locking shard
          # wallclock.  t/327 HW workload / t/328 MG-B write perf / t/329 MG-D
          # heap-ITL WAL measure / t/330 production-bench-subset / t/331 4-node
          # reconfig fault matrix.
          - { name: stage5-integrated-acceptance, ranges: "327-331", unit: false, regress: false }
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install Linux dependencies
        run: |
          sudo apt-get update -y
          sudo apt-get install -y --no-install-recommends \
            build-essential ccache \
            libreadline-dev zlib1g-dev libicu-dev \
            liblz4-dev libzstd-dev libssl-dev libxml2-dev \
            libipc-run-perl pkg-config

      - name: Restore ccache
        uses: actions/cache@v4
        with:
          path: .ccache
          key: ${{ runner.os }}-${{ github.job }}-${{ matrix.shard.name }}-ccache-${{ github.ref_name }}
          restore-keys: |
            ${{ runner.os }}-${{ github.job }}-${{ matrix.shard.name }}-ccache-
            ${{ runner.os }}-${{ github.job }}-ccache-
            ${{ runner.os }}-ccache-

      - name: Configure ccache
        run: |
          ccache --max-size=700M
          ccache --zero-stats

      - name: Configure (--enable-cluster)
        run: |
          CC="ccache gcc" ./configure \
            --prefix=$HOME/pgrac-install \
            --enable-cassert --enable-debug \
            --with-openssl --with-icu --with-lz4 --with-zstd \
            --enable-cluster --enable-tap-tests

      - name: Build and install
        run: |
          make -j$(getconf _NPROCESSORS_ONLN)
          ccache --show-stats
          make install

      - name: Run cluster_unit
        if: matrix.shard.unit
        run: make -C src/test/cluster_unit check

      - name: Run cluster_regress (foundation-core shard only)
        if: matrix.shard.regress
        run: make -C src/test/cluster_regress check

      - name: Run cluster_tap shard ${{ matrix.shard.name }}
        env:
          # Harmless to set everywhere.  The medium Stage 2 acceptance job
          # owns t/202; keep the env here so manual shard experiments behave
          # the same if t/202 is temporarily included in a range.
          STAGE2_PERF_HARD_GATE: '1'
        run: |
          SEL=$(cd src/test/cluster_tap && ranges="${{ matrix.shard.ranges }}"; for f in t/*.pl; do \
            n=$(basename "$f" .pl | cut -d_ -f1); \
            d=$((10#$n)); \
            include=0; \
            for r in $ranges; do \
              lo=${r%-*}; hi=${r#*-}; \
              lo_d=$((10#$lo)); hi_d=$((10#$hi)); \
              if [ "$d" -ge "$lo_d" ] && [ "$d" -le "$hi_d" ]; then \
                include=1; break; \
              fi; \
            done; \
            if [ "$include" -eq 1 ]; then \
              printf '%s ' "$f"; \
            fi; \
          done)
          echo "shard ${{ matrix.shard.name }} files: $SEL"
          test -n "$SEL" || { echo "ERROR: empty shard selection"; exit 1; }
          make -C src/test/cluster_tap check PROVE_TESTS="$SEL"

      - name: Upload cluster_tap diffs on failure
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: nightly-cluster-tap-diffs-${{ matrix.shard.name }}
          path: |
            src/test/cluster_tap/tmp_check/log/
          if-no-files-found: ignore
          retention-days: 14

  # ====================================================================
  # Tier 2.1c — macOS Build+Test (smoke)
  # --------------------------------------------------------------------
  # macOS keeps build + PG 219 + cluster_unit + cluster_regress + a curated
  # cluster_tap SMOKE subset only (the full cluster_tap on macOS was the
  # ~22min nightly bottleneck; Linux runs the full suite, sharded, above).
  # The smoke deliberately INCLUDES the concurrency-sensitive spec-3.19
  # 234/235 (macOS shmem/IPC behaviour is exactly where those matter most).
  # ====================================================================
  build-test-macos:
    name: macOS Build+Test (smoke)
    runs-on: macos-latest
    timeout-minutes: 45
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install macOS dependencies
        run: |
          brew install readline icu4c lz4 zstd openssl@3 libxml2 cpanminus pkg-config ccache || true
          cpanm --local-lib=$HOME/perl5 --notest IPC::Run IO::Tty
          {
            echo "PERL5LIB=$HOME/perl5/lib/perl5"
            echo "PKG_CONFIG_PATH=$(brew --prefix icu4c)/lib/pkgconfig:$(brew --prefix openssl@3)/lib/pkgconfig:$(brew --prefix lz4)/lib/pkgconfig:$(brew --prefix zstd)/lib/pkgconfig"
            echo "LDFLAGS=-L$(brew --prefix readline)/lib -L$(brew --prefix zstd)/lib -L$(brew --prefix lz4)/lib -L$(brew --prefix openssl@3)/lib -L$(brew --prefix icu4c)/lib"
            echo "CPPFLAGS=-I$(brew --prefix readline)/include -I$(brew --prefix zstd)/include -I$(brew --prefix lz4)/include -I$(brew --prefix openssl@3)/include -I$(brew --prefix icu4c)/include"
          } >> $GITHUB_ENV

      - name: Restore ccache
        uses: actions/cache@v4
        with:
          path: .ccache
          key: ${{ runner.os }}-${{ github.job }}-ccache-${{ github.ref_name }}
          restore-keys: |
            ${{ runner.os }}-${{ github.job }}-ccache-
            ${{ runner.os }}-ccache-

      - name: Configure ccache
        run: |
          ccache --max-size=700M
          ccache --zero-stats

      - name: Configure (--enable-cluster)
        run: |
          CC="ccache clang" ./configure \
            --prefix=$HOME/pgrac-install \
            --enable-cassert --enable-debug \
            --with-openssl --with-icu --with-lz4 --with-zstd \
            --enable-cluster --enable-tap-tests

      - name: Build and install
        run: |
          make -j$(sysctl -n hw.ncpu)
          ccache --show-stats
          make install

      - name: Run PG regression tests (219 tests)
        run: make check

      - name: Run cluster smoke (unit + regress + curated TAP)
        run: |
          make -C src/test/cluster_unit check
          make -C src/test/cluster_regress check
          SMOKE="t/001_single_node_smoke.pl t/005_wait_events_catalog.pl t/020_shmem_registry.pl t/068_wal_xl_scn.pl t/215_cluster_3_9_cr_construction.pl t/219_cluster_3_11_durable_tt_slot.pl t/226_stage3_mvcc_acceptance_capability.pl t/234_cluster_3_19_dml_boundary.pl t/235_cluster_3_19_snapshot_isolation.pl"
          make -C src/test/cluster_tap check PROVE_TESTS="$SMOKE"

      - name: Upload regression diffs on failure
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: nightly-regression-diffs-macos
          path: |
            src/test/regress/regression.diffs
            src/test/regress/regression.out
            src/test/regress/log/
            src/test/cluster_tap/tmp_check/log/
          if-no-files-found: ignore
          retention-days: 14

  # ====================================================================
  # Tier 2.2 — Stage 2 acceptance medium representative job
  # --------------------------------------------------------------------
  # Self-contained Ubuntu job.  It intentionally rebuilds/install locally
  # instead of assuming workspace or install artifacts from other jobs.
  # Runs the Stage 2 acceptance TAPs with longer knobs than fast-gate and
  # uploads the generated JSON reports.
  # ====================================================================
  stage2-acceptance-medium:
    name: Stage 2 acceptance medium (Ubuntu)
    runs-on: ubuntu-latest
    timeout-minutes: 90
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install Linux dependencies
        run: |
          sudo apt-get update -y
          sudo apt-get install -y --no-install-recommends \
            build-essential ccache \
            libreadline-dev zlib1g-dev libicu-dev \
            liblz4-dev libzstd-dev libssl-dev libxml2-dev \
            libipc-run-perl pkg-config

      - name: Restore ccache
        uses: actions/cache@v4
        with:
          path: .ccache
          key: ${{ runner.os }}-${{ github.job }}-ccache-${{ github.ref_name }}
          restore-keys: |
            ${{ runner.os }}-${{ github.job }}-ccache-
            ${{ runner.os }}-ccache-

      - name: Configure ccache
        run: |
          ccache --max-size=700M
          ccache --zero-stats

      - name: Configure (--enable-cluster)
        run: |
          CC="ccache gcc" ./configure \
            --prefix=$HOME/pgrac-install \
            --enable-cassert --enable-debug \
            --with-openssl --with-icu --with-lz4 --with-zstd \
            --enable-cluster --enable-tap-tests

      - name: Build and install
        run: |
          make -j$(getconf _NPROCESSORS_ONLN)
          ccache --show-stats
          make install

      - name: Run Stage 2 acceptance TAP matrix
        env:
          STAGE2_TAG: ${{ github.ref_name }}
          STAGE2_PGBENCH_SECONDS: "20"
          STAGE2_WORKLOAD_SECONDS: "1"
          STAGE2_WORKLOAD_ITERATIONS: "20"
        run: |
          make -C src/test/cluster_tap check \
            PROVE_TESTS="t/200_stage2_acceptance_capability.pl t/201_stage2_acceptance_fault_matrix.pl t/202_stage2_acceptance_perf_smoke.pl"

      - name: Upload Stage 2 acceptance reports
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: stage2-acceptance-medium-${{ github.run_id }}
          path: |
            src/test/cluster_tap/tmp/stage2-acceptance-*.json
            src/test/cluster_tap/tmp_check/log/
          if-no-files-found: ignore
          retention-days: 30

  # ====================================================================
  # Tier 2.3 — Stage 3 MVCC acceptance medium representative job
  # --------------------------------------------------------------------
  # Self-contained Ubuntu job (rebuilds/install locally).  Runs the
  # spec-3.17 Stage 3 acceptance TAPs — capability cross-cutting (t/226)
  # and the workload perf smoke (t/227, pgbench) — with longer knobs than
  # fast-gate, plus the static contract unit binary, and uploads the
  # generated JSON reports.  All perf numbers are report-only (the Stage 3
  # performance bar is owned by spec-3.18);  this job gates only on the
  # functional/correctness assertions.
  # ====================================================================
  stage3-acceptance-medium:
    name: Stage 3 acceptance medium (Ubuntu)
    runs-on: ubuntu-latest
    timeout-minutes: 90
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install Linux dependencies
        run: |
          sudo apt-get update -y
          sudo apt-get install -y --no-install-recommends \
            build-essential ccache \
            libreadline-dev zlib1g-dev libicu-dev \
            liblz4-dev libzstd-dev libssl-dev libxml2-dev \
            libipc-run-perl pkg-config

      - name: Restore ccache
        uses: actions/cache@v4
        with:
          path: .ccache
          key: ${{ runner.os }}-${{ github.job }}-ccache-${{ github.ref_name }}
          restore-keys: |
            ${{ runner.os }}-${{ github.job }}-ccache-
            ${{ runner.os }}-ccache-

      - name: Configure ccache
        run: |
          ccache --max-size=700M
          ccache --zero-stats

      - name: Configure (--enable-cluster)
        run: |
          CC="ccache gcc" ./configure \
            --prefix=$HOME/pgrac-install \
            --enable-cassert --enable-debug \
            --with-openssl --with-icu --with-lz4 --with-zstd \
            --enable-cluster --enable-tap-tests

      - name: Build and install
        run: |
          make -j$(getconf _NPROCESSORS_ONLN)
          ccache --show-stats
          make install

      - name: Run Stage 3 static contract unit binary
        run: make -C src/test/cluster_unit check

      - name: Run Stage 3 acceptance TAP matrix
        env:
          PGRAC_TAG: ${{ github.ref_name }}
          STAGE3_PGBENCH_SECONDS: "20"
        run: |
          make -C src/test/cluster_tap check \
            PROVE_TESTS="t/226_stage3_mvcc_acceptance_capability.pl t/227_stage3_mvcc_workload_smoke.pl"

      - name: Upload Stage 3 acceptance reports
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: stage3-acceptance-medium-${{ github.run_id }}
          path: |
            src/test/cluster_tap/tmp/stage3-acceptance-*.json
            src/test/cluster_tap/tmp_check/log/
          if-no-files-found: ignore
          retention-days: 30