ci3.yml

# CI for Aztec. At a high-level, runs ./bootstrap.sh ci in root. See root README.md for more details.
# Only for internal devs. For external devs, see ci3-external.yml.
#
# CAREFUL! We use "exec" a lot to ensure signal propagation to the child process, to allow proper ec2 cleanup.
name: CI3

on:
  workflow_dispatch:
  push:
    tags:
      - "v*"
  pull_request:
    types: [opened, synchronize, reopened, ready_for_review, labeled]
  merge_group: {}

concurrency:
  # Allow full concurrency for merge-train PRs, one-run-per-branch for everything else.
  group: ci3-${{ (startsWith(github.event.pull_request.head.ref, 'merge-train/') && github.run_id) || github.event.merge_group.head_ref || github.ref_name }}
  cancel-in-progress: true

jobs:
  # The main CI job for Aztec pull requests. It gatekeeps merges to next, intended for the merge queue.
  #
  # This orchestrates AWS EC2 spot instances then remotely runs ./bootstrap.sh
  # The ci-full label enables heavier tests. For other labels, see ci3.sh.
  #
  #            Standard PR)                      Merge Queue
  #                  |                                 |
  #      |  1x AMD64 EC2 Spot  |           |  [ 4x AMD64 EC2 Spot ]  |
  #                                        |  [ 1x ARM64 EC2 Spot ]  |
  #                                                    |
  #                                           Filters out "accidental"
  #                                           successes, but non-determinism
  #                                           can still cause flaky tests.
  #
  # If you suspect a flaky test, look for people recently reporting similar things.
  # Otherwise, spend time ensuring it is not your PR. Spend some time attempting to fix it.
  # Try not to just increase timeouts. Post either about your fix or your efforts where
  # engineers can broadly see it.
  ci:
    runs-on: ubuntu-latest
    # exclusive with ci3-external.yml: never run on forks
    # (github.event.pull_request.head.repo.fork resolves to nil if not a pull request)
    if: github.event.pull_request.head.repo.fork != true && (github.event.pull_request.draft == false || contains(github.event.pull_request.labels.*.name, 'ci-draft'))
    environment: ${{ startsWith(github.ref, 'refs/tags/v') && 'master' || '' }}
    permissions:
      id-token: write   # required for OIDC assume-role with AWS
      contents: read
    steps:
      - name: Remove wakeup label
        if: contains(github.event.pull_request.labels.*.name, 'ci-wakeup-pr-after-merge')
        env:
          GH_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
        run: gh pr edit ${{ github.event.pull_request.number }} --remove-label ci-wakeup-pr-after-merge --repo ${{ github.repository }} || true

      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
        with:
          # The commit to checkout. We want our actual commit, and not the result of merging the PR to the target.
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
          # Fetch PR commits depth (we'll deepen by 1 in squash script if needed)
          fetch-depth: ${{ github.event.pull_request.commits || 1 }}
          persist-credentials: true   # Required for bootstrap_ec2's git fetch

      - name: Configure AWS credentials (OIDC)
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
          aws-region: us-east-2
          role-session-name: ci3-${{ github.run_id }}
          role-duration-seconds: 7200 # 2h – covers max AWS_SHUTDOWN_TIME (90 min ARM) + 30 min buffer

      - name: Determine CI Mode
        env:
          MERGE_GROUP_BASE_REF: ${{ github.event.merge_group.base_ref }}
          PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
          GITHUB_REF_NAME: ${{ github.ref_name }}
          GITHUB_TOKEN: ${{ github.token }}
          PR_LABELS_JSON: ${{ toJson(github.event.pull_request.labels.*.name) }}
        run: |
          # Parse labels from JSON env var to avoid shell injection via label names
          mapfile -t LABELS < <(echo "$PR_LABELS_JSON" | jq -r '.[]')
          ./.github/ci3_labels_to_env.sh "${LABELS[@]}"

      - name: Run
        env:
          CI_USE_SSH: ${{ (vars.CI_USE_SSH == '1' && !contains(github.event.pull_request.labels.*.name, 'ci-ssm')) && '1' || '0' }}
          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
          NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
          AZTEC_TOOLCHAIN_DEFAULT_MAJOR_VERSION: ${{ vars.AZTEC_TOOLCHAIN_DEFAULT_MAJOR_VERSION }}
          # For automatic documentation updates via Claude Code
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          # Nightly test env vars.
          EXTERNAL_ETHEREUM_HOSTS: "https://json-rpc.${{ secrets.GCP_SEPOLIA_URL }}?key=${{ secrets.GCP_SEPOLIA_API_KEY }},${{ secrets.INFURA_SEPOLIA_URL }}"
          EXTERNAL_ETHEREUM_CONSENSUS_HOST: "https://beacon.${{ secrets.GCP_SEPOLIA_URL }}"
          EXTERNAL_ETHEREUM_CONSENSUS_HOST_API_KEY: ${{ secrets.GCP_SEPOLIA_API_KEY }}
          EXTERNAL_ETHEREUM_CONSENSUS_HOST_API_KEY_HEADER: "X-goog-api-key"
          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
          MERGE_GROUP_BASE_REF: ${{ github.event.merge_group.base_ref }}
          PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
          PR_HEAD_REF: ${{ github.event.pull_request.head.ref }}
          PR_COMMITS: ${{ github.event.pull_request.commits }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
          GITHUB_REF_NAME: ${{ github.ref_name }}
          GITHUB_ACTOR: ${{ github.actor }}
          CI3_INSTANCE_PROFILE_NAME: ${{ secrets.CI3_INSTANCE_PROFILE_NAME }}
          CI3_SECURITY_GROUP_ID: ${{ secrets.CI3_SECURITY_GROUP_ID }}
          AWS_OIDC_ROLE_ARN: ${{ secrets.AWS_OIDC_ROLE_ARN }}
        # NOTE: $CI_MODE is set in the Determine CI Mode step.
        run: ./.github/ci3.sh $CI_MODE

      - name: Post-Actions
        env:
          SHOULD_SQUASH_MERGE: ${{ contains(github.event.pull_request.labels.*.name, 'ci-squash-and-merge') && '1' || '0' }}
          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
          PR_HEAD_REF: ${{ github.event.pull_request.head.ref }}
          PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
          PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
        run: ./.github/ci3_success.sh

      - name: Notify Slack on backport CI failure
        if: failure() && startsWith(github.event.pull_request.head.ref, 'backport-to-') && endsWith(github.event.pull_request.head.ref, '-staging')
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
          PR_HTML_URL: ${{ github.event.pull_request.html_url }}
          PR_NUM: ${{ github.event.pull_request.number }}
          PR_TITLE: ${{ github.event.pull_request.title }}
          RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          if [ -n "${SLACK_BOT_TOKEN}" ]; then
            TEXT="CI3 failed on backport PR: <${PR_HTML_URL}|#${PR_NUM} - ${PR_TITLE}>\n<${RUN_URL}|View Run>"
            curl -X POST https://slack.com/api/chat.postMessage \
              -H "Authorization: Bearer $SLACK_BOT_TOKEN" \
              -H "Content-type: application/json" \
              --data "$(jq -n --arg c "#backports" --arg t "$TEXT" '{channel:$c, text:$t}')"
          fi

      - name: Upload benchmarks
        if: env.SHOULD_UPLOAD_BENCHMARKS == '1'
        uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
        with: &ci_benchmark_args
          name: Aztec Benchmarks
          benchmark-data-dir-path: "bench/${{ env.BENCH_BRANCH }}"
          tool: "customSmallerIsBetter"
          output-file-path: ./bench-out/bench.json
          gh-repository: github.com/AztecProtocol/benchmark-page-data
          github-token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
          auto-push: true
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
          alert-threshold: "105%"
          comment-on-alert: false
          fail-on-alert: false
          max-items-in-chart: 100

  # Validates that a nightly tag's embedded date matches today's UTC date.
  # Prevents stale or replayed nightly tags from triggering scenario tests.
  validate-nightly-tag:
    runs-on: ubuntu-latest
    if: startsWith(github.ref, 'refs/tags/v') && contains(github.ref_name, '-nightly.')
    outputs:
      is_current: ${{ steps.check.outputs.is_current }}
    steps:
      - name: Check nightly tag date matches today (UTC)
        id: check
        run: |
          tag_date=$(echo "${{ github.ref_name }}" | grep -oP '(?<=-nightly\.)\d{8}')
          today_utc=$(date -u +%Y%m%d)
          echo "Tag date: $tag_date"
          echo "Today (UTC): $today_utc"
          if [[ "$tag_date" == "$today_utc" ]]; then
            echo "is_current=true" >> $GITHUB_OUTPUT
          else
            echo "::warning::Nightly tag date ($tag_date) does not match today's UTC date ($today_utc). Skipping scenario tests."
          fi

  # End-to-end tests that target a network deployment.
  # We run this every release (at minimum, nightly), or when explicitly requested.
  # This task runs against a real testnet deployment. This uses resources on GCP (not AWS, thank free credit incentives).
  # Runs two test sets in parallel.
  ci-network-scenario:
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      contents: read
    strategy:
      fail-fast: false
      matrix:
        test_set: ["1", "2"]
    # We run on current nightly tags only, or when the ci-network-scenario label is present in a PR.
    needs: [ci, validate-nightly-tag]
    if: |
      always()
      && (needs.ci.result == 'success' || needs.ci.result == 'skipped')
      && github.event.pull_request.head.repo.fork != true
      && github.event.pull_request.draft == false
      && (
        needs.validate-nightly-tag.outputs.is_current == 'true'
        || contains(github.event.pull_request.labels.*.name, 'ci-network-scenario')
      )
    steps:
      - name: Remove label (one-time use)
        if: github.event.pull_request && contains(github.event.pull_request.labels.*.name, 'ci-network-scenario')
        env:
          GH_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
        run: gh pr edit ${{ github.event.pull_request.number }} --remove-label ci-network-scenario --repo ${{ github.repository }} || true

      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}

      - name: Configure AWS credentials (OIDC)
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
          aws-region: us-east-2
          role-session-name: ci3-network-scenario-${{ github.run_id }}
          role-duration-seconds: 23400 # 6.5h – covers AWS_SHUTDOWN_TIME (360 min) + 30 min buffer

      - name: Run Network Scenarios
        timeout-minutes: 350
        env:
          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
          # For pushing docker images (only for PR label, otherwise we use the tag)
          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
          CI3_INSTANCE_PROFILE_NAME: ${{ secrets.CI3_INSTANCE_PROFILE_NAME }}
          CI3_SECURITY_GROUP_ID: ${{ secrets.CI3_SECURITY_GROUP_ID }}
          RUN_ID: ${{ github.run_id }}
          AWS_SHUTDOWN_TIME: 360
          NO_SPOT: 1
          GIT_REF: ${{ github.ref }}
          GIT_REF_NAME: ${{ github.ref_name }}
          HEAD_REF: ${{ github.head_ref || github.ref_name }}
        run: |
          # For release tags, use the release image; for PRs, omit to build and push to aztecdev
          if [[ "$GIT_REF" == refs/tags/v* ]]; then
            tag="$GIT_REF_NAME"
            tag="${tag#v}"
            major_version=$(./ci3/semver major "$tag")
            namespace="v${major_version}-scenario"
            docker_image="aztecprotocol/aztec:${tag}"
          else
            # branch name
            namespace=pr-$(echo "$HEAD_REF" | sed 's/[^a-z0-9-]/-/g' | cut -c1-20 | sed 's/-*$//')
            docker_image=""
          fi
          echo "NAMESPACE=$namespace" >> $GITHUB_ENV
          set -x # print next line
          ./.github/ci3.sh network-scenarios next-scenario "$namespace" "$docker_image" "${{ matrix.test_set }}"

      - name: Cleanup network resources
        # Clean up if this is a CI label or nightly.
        if: always() && (!startsWith(github.ref, 'refs/tags/v') || contains(github.ref_name, '-nightly.'))
        env:
          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
          CI3_INSTANCE_PROFILE_NAME: ${{ secrets.CI3_INSTANCE_PROFILE_NAME }}
          CI3_SECURITY_GROUP_ID: ${{ secrets.CI3_SECURITY_GROUP_ID }}
          NO_SPOT: 1
        run: |
          ./.github/ci3.sh network-teardown next-scenario "${NAMESPACE}-${{ matrix.test_set }}" || true

      #############
      # Benchmarks
      #############
      - name: Download deploy benchmarks
        if: always() && startsWith(github.ref, 'refs/tags/v')
        run: |
          if ./ci.sh gh-deploy-bench; then
            echo "ENABLE_DEPLOY_BENCH=1" >> $GITHUB_ENV
          fi

      - name: Upload benchmarks
        if: env.ENABLE_DEPLOY_BENCH == '1'
        uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
        with: *ci_benchmark_args

      - name: Notify Slack and dispatch ClaudeBox on failure
        if: failure() && startsWith(github.ref, 'refs/tags/v')
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
          GIT_REF: ${{ github.ref }}
          RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
          TEST_SET: ${{ matrix.test_set }}
        run: |
          ./ci3/slack_notify_with_claudebox_kickoff "#alerts-next-scenario" \
            "Nightly Spartan Scenario FAILED in ci3.yml (set ${TEST_SET}, nightly tag ${GIT_REF}): <${RUN_URL}|View Run> (🤖)" \
            "Nightly Spartan Scenario tests failed (matrix set ${TEST_SET}, nightly tag ${GIT_REF}). CI run: ${RUN_URL}. Investigate the scenario test failure and identify the root cause." \
            --link "$RUN_URL"

  # Spartan network benchmarks triggered on-demand from a PR label.
  # Runs TPS and proving benchmarks in parallel, uploads results to a PR-specific
  # path on the benchmark dashboard so nightly (bench/next) is never affected.
  # One-time use: label is removed after the job runs.
  ci-network-bench:
    name: ${{ matrix.bench_type }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - bench_type: benchmark
            ci3_cmd: network-bench
            scenario: tps-scenario
            namespace_suffix: bench
            download_cmd: gh-spartan-bench
            timeout: 300
          - bench_type: proving-benchmark
            ci3_cmd: network-proving-bench
            scenario: prove-n-tps-fake
            namespace_suffix: proving-bench
            download_cmd: gh-spartan-proving-bench
            timeout: 240
    needs: ci
    if: github.event.pull_request.head.repo.fork != true && github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci-network-bench') && (needs.ci.result == 'success' || needs.ci.result == 'skipped')
    steps:
      - name: Remove label (one-time use)
        env:
          GH_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
        run: gh pr edit ${{ github.event.pull_request.number }} --remove-label ci-network-bench --repo ${{ github.repository }} || true

      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}

      - name: Run Network Benchmarks
        timeout-minutes: ${{ matrix.timeout }}
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
          # For pushing docker images built from the PR
          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
          RUN_ID: ${{ github.run_id }}
          AWS_SHUTDOWN_TIME: ${{ matrix.timeout }}
          NO_SPOT: 1
        run: |
          namespace=pr-${{ github.event.pull_request.number }}-${{ matrix.namespace_suffix }}
          echo "NAMESPACE=$namespace" >> $GITHUB_ENV
          set -x
          ./.github/ci3.sh ${{ matrix.ci3_cmd }} ${{ matrix.scenario }} "$namespace"

      - name: Cleanup network resources
        if: always()
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
          NO_SPOT: 1
        run: ./.github/ci3.sh network-teardown ${{ matrix.scenario }} "${NAMESPACE}" || true

      - name: Download benchmarks
        if: always()
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        run: |
          if ./ci.sh ${{ matrix.download_cmd }}; then
            echo "ENABLE_DEPLOY_BENCH=1" >> $GITHUB_ENV
          fi

      - name: Upload benchmarks
        if: always() && env.ENABLE_DEPLOY_BENCH == '1'
        uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
        with:
          name: Spartan
          benchmark-data-dir-path: "bench/pr-${{ github.event.pull_request.number }}"
          tool: "customSmallerIsBetter"
          output-file-path: ./bench-out/bench.json
          gh-repository: github.com/AztecProtocol/benchmark-page-data
          github-token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
          auto-push: true
          ref: ${{ github.event.pull_request.head.sha || github.sha }}
          alert-threshold: "120%"
          comment-on-alert: false
          fail-on-alert: false
          max-items-in-chart: 100

  # KIND-based e2e tests that run on a local Kubernetes cluster.
  # One-time use: label is removed after the job runs.
  ci-network-kind:
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      contents: read
    needs: ci
    if: github.event.pull_request.head.repo.fork != true && github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci-network-kind')
    timeout-minutes: 180 # 3 hours for KIND tests
    steps:
      - name: Remove label
        env:
          GH_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
        run: gh pr edit ${{ github.event.pull_request.number }} --remove-label ci-network-kind --repo ${{ github.repository }}

      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}

      - name: Configure AWS credentials (OIDC)
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
          aws-region: us-east-2
          role-session-name: ci3-network-kind-${{ github.run_id }}
          role-duration-seconds: 12600 # 3.5h – covers AWS_SHUTDOWN_TIME (180 min) + 30 min buffer

      - name: Run KIND Test
        env:
          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
          CI3_INSTANCE_PROFILE_NAME: ${{ secrets.CI3_INSTANCE_PROFILE_NAME }}
          CI3_SECURITY_GROUP_ID: ${{ secrets.CI3_SECURITY_GROUP_ID }}
          RUN_ID: ${{ github.run_id }}
          AWS_SHUTDOWN_TIME: 180
        run: |
          ./.github/ci3.sh network-tests-kind

  # Backwards compatibility e2e tests.
  # Runs e2e tests with contract artifacts from every prior stable release to validate
  # that new client code works with old contract artifacts ("new pxe / old contracts").
  # Blocking for stable/RC releases: ci-release-publish requires this job to pass before
  # publishing. Observational for nightlies: runs, but continue-on-error keeps the workflow
  # green and ci-release-publish's condition publishes nightlies regardless of the result.
  # Escape hatch: ci-skip-compat-e2e label makes failures non-blocking on release PRs.
  ci-compat-e2e:
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      contents: read
    needs: [ci]
    if: |
      always()
      && (needs.ci.result == 'success' || needs.ci.result == 'skipped')
      && github.event.pull_request.head.repo.fork != true
      && github.event.pull_request.draft == false
      && (
        (startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-commit.'))
        || contains(github.event.pull_request.labels.*.name, 'ci-compat-e2e')
        || contains(github.event.pull_request.labels.*.name, 'ci-release-pr')
      )
    # Non-blocking for nightlies and when ci-skip-compat-e2e escape hatch is applied.
    continue-on-error: ${{ contains(github.ref_name, '-nightly.') || contains(github.event.pull_request.labels.*.name, 'ci-skip-compat-e2e') }}
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
        with:
          ref: ${{ github.event.pull_request.head.sha || github.sha }}

      - name: Configure AWS credentials (OIDC)
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
          aws-region: us-east-2
          role-session-name: ci3-compat-e2e-${{ github.run_id }}
          role-duration-seconds: 21600 # 6h – covers AWS_SHUTDOWN_TIME (300 min) + 60 min buffer

      - name: Run Backwards Compatibility E2E Tests
        timeout-minutes: 330
        env:
          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
          CI3_INSTANCE_PROFILE_NAME: ${{ secrets.CI3_INSTANCE_PROFILE_NAME }}
          CI3_SECURITY_GROUP_ID: ${{ secrets.CI3_SECURITY_GROUP_ID }}
          RUN_ID: ${{ github.run_id }}
          AWS_SHUTDOWN_TIME: 300
        run: ./.github/ci3.sh compat-e2e

  # Publishes the release (npm, Docker, GitHub release, aztec-up scripts, etc.).
  # Gated on ci-compat-e2e: a compat regression blocks stable/RC publishing. Nightlies
  # publish regardless — compat-e2e runs there observationally. Dev `-commit.` tags from
  # the ci-release-pr flow never reach this job (they are not real releases).
  ci-release-publish:
    runs-on: ubuntu-latest
    environment: master
    permissions:
      id-token: write
      contents: read
    needs: [ci, ci-compat-e2e]
    if: |
      startsWith(github.ref, 'refs/tags/v')
      && !contains(github.ref_name, '-commit.')
      && needs.ci.result == 'success'
      && (
        contains(github.ref_name, '-nightly.')
        || needs.ci-compat-e2e.result == 'success'
      )
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
        with:
          ref: ${{ github.sha }}

      - name: Configure AWS credentials (OIDC)
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
          aws-region: us-east-2
          role-session-name: ci3-release-publish-${{ github.run_id }}
          role-duration-seconds: 21600

      - name: Run Release Publish
        env:
          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
          NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
          CI3_INSTANCE_PROFILE_NAME: ${{ secrets.CI3_INSTANCE_PROFILE_NAME }}
          CI3_SECURITY_GROUP_ID: ${{ secrets.CI3_SECURITY_GROUP_ID }}
          RUN_ID: ${{ github.run_id }}
        run: ./.github/ci3.sh release-publish