feat: merge-train/fairies (#22936)

See
[merge-train-readme.md](https://github.com/AztecProtocol/aztec-packages/blob/next/.github/workflows/merge-train-readme.md).
This is a merge-train.

Author: AztecBot

.github/ci3_labels_to_env.sh

@@ -37,6 +37,11 @@ function main {
     echo "NO_FAIL_FAST=1" >> $GITHUB_ENV
   fi
 
+  # Handle skip-compat-e2e label (escape hatch for backwards compat test failures on release PRs)
+  if has_label "ci-skip-compat-e2e"; then
+    echo "SKIP_COMPAT_E2E=1" >> $GITHUB_ENV
+  fi
+
   # Determine CI mode based on event, labels, and target branch
   local ci_mode
   if [ "${GITHUB_EVENT_NAME:-}" == "merge_group" ] || has_label "ci-merge-queue"; then

.github/workflows/ci3.yml

@@ -445,3 +445,108 @@ jobs:
           AWS_SHUTDOWN_TIME: 180
         run: |
           ./.github/ci3.sh network-tests-kind
+
+  # Backwards compatibility e2e tests.
+  # Runs e2e tests with contract artifacts from every prior stable release to validate
+  # that new client code works with old contract artifacts ("new pxe / old contracts").
+  # Blocking for stable/RC releases: ci-release-publish requires this job to pass before
+  # publishing. Observational for nightlies: runs, but continue-on-error keeps the workflow
+  # green and ci-release-publish's condition publishes nightlies regardless of the result.
+  # Escape hatch: ci-skip-compat-e2e label makes failures non-blocking on release PRs.
+  ci-compat-e2e:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    needs: [ci]
+    if: |
+      always()
+      && (needs.ci.result == 'success' || needs.ci.result == 'skipped')
+      && github.event.pull_request.head.repo.fork != true
+      && github.event.pull_request.draft == false
+      && (
+        (startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-commit.'))
+        || contains(github.event.pull_request.labels.*.name, 'ci-compat-e2e')
+        || contains(github.event.pull_request.labels.*.name, 'ci-release-pr')
+      )
+    # Non-blocking for nightlies and when ci-skip-compat-e2e escape hatch is applied.
+    continue-on-error: ${{ contains(github.ref_name, '-nightly.') || contains(github.event.pull_request.labels.*.name, 'ci-skip-compat-e2e') }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+
+      - name: Configure AWS credentials (OIDC)
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
+          aws-region: us-east-2
+          role-session-name: ci3-compat-e2e-${{ github.run_id }}
+          role-duration-seconds: 21600 # 6h – covers AWS_SHUTDOWN_TIME (300 min) + 60 min buffer
+
+      - name: Run Backwards Compatibility E2E Tests
+        timeout-minutes: 330
+        env:
+          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
+          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
+          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          CI3_INSTANCE_PROFILE_NAME: ${{ secrets.CI3_INSTANCE_PROFILE_NAME }}
+          CI3_SECURITY_GROUP_ID: ${{ secrets.CI3_SECURITY_GROUP_ID }}
+          RUN_ID: ${{ github.run_id }}
+          AWS_SHUTDOWN_TIME: 300
+        run: ./.github/ci3.sh compat-e2e
+
+  # Publishes the release (npm, Docker, GitHub release, aztec-up scripts, etc.).
+  # Gated on ci-compat-e2e: a compat regression blocks stable/RC publishing. Nightlies
+  # publish regardless — compat-e2e runs there observationally. Dev `-commit.` tags from
+  # the ci-release-pr flow never reach this job (they are not real releases).
+  ci-release-publish:
+    runs-on: ubuntu-latest
+    environment: master
+    permissions:
+      id-token: write
+      contents: read
+    needs: [ci, ci-compat-e2e]
+    if: |
+      startsWith(github.ref, 'refs/tags/v')
+      && !contains(github.ref_name, '-commit.')
+      && needs.ci.result == 'success'
+      && (
+        contains(github.ref_name, '-nightly.')
+        || needs.ci-compat-e2e.result == 'success'
+      )
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          ref: ${{ github.sha }}
+
+      - name: Configure AWS credentials (OIDC)
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
+          aws-region: us-east-2
+          role-session-name: ci3-release-publish-${{ github.run_id }}
+          role-duration-seconds: 21600
+
+      - name: Run Release Publish
+        env:
+          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
+          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
+          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
+          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          CI3_INSTANCE_PROFILE_NAME: ${{ secrets.CI3_INSTANCE_PROFILE_NAME }}
+          CI3_SECURITY_GROUP_ID: ${{ secrets.CI3_SECURITY_GROUP_ID }}
+          RUN_ID: ${{ github.run_id }}
+        run: ./.github/ci3.sh release-publish

bootstrap.sh

@@ -823,6 +823,17 @@ case "$cmd" in
   # RELEASES #
   ############
   "ci-release")
+    # Verification build for a release tag. Does NOT publish — publishing happens in
+    # ci-release-publish, gated on ci-compat-e2e so a compat regression blocks the release.
+    export CI=1
+    export USE_TEST_CACHE=1
+    if ! semver check $REF_NAME; then
+      exit 1
+    fi
+    build
+    ;;
+  "ci-release-publish")
+    # Actual publish step. `build` cache-hits against ci-release's build of the same commit.
     export CI=1
     export USE_TEST_CACHE=1
     if ! semver check $REF_NAME; then
@@ -902,6 +913,82 @@ case "$cmd" in
     build
     yarn-project/end-to-end/bootstrap.sh avm_check_circuit
     ;;
+  #############################################
+  # BACKWARDS COMPATIBILITY E2E TESTS         #
+  #############################################
+  "ci-compat-e2e")
+    # Runs e2e tests with contract artifacts from every prior stable release since 4.2.0 (version where we committed to
+    # backwards compatibility). This Validates that old contract artifacts work on current release.
+    export CI=1
+    export USE_TEST_CACHE=0
+    export CI_FULL=0
+    export NO_FAIL_FAST=1
+
+    build
+
+    # TODO: bump when v5 commits to backwards-compatible contract artifacts.
+    #   compat_major:       major version that has compat guarantees today.
+    #   compat_min_version: earliest stable tag of that major to test against
+    #                       (artifacts before this are incompatible due to oracle interface changes).
+    compat_major="4"
+    compat_min_version="4.2.0"
+
+    # Get current major version.
+    current_version=$(jq -r '."."' .release-please-manifest.json)
+    major=$(semver major "$current_version")
+    if [ "$major" != "$compat_major" ]; then
+      echo "Compat e2e tests only apply to v${compat_major}. Current major: v${major}. Skipping."
+      exit 0
+    fi
+    min_version="$compat_min_version"
+
+    # Fetch tags (EC2 clone may not have them). Fail loud: a silent fetch failure plus an empty
+    # tag list would publish a real release with zero compat coverage.
+    if ! git fetch origin 'refs/tags/v*:refs/tags/v*'; then
+      echo "ERROR: failed to fetch release tags." >&2
+      exit 1
+    fi
+
+    # Discover stable tags for this major version (no prerelease suffixes).
+    versions=()
+    while IFS= read -r tag; do
+      ver=${tag#v}
+      # Include only versions >= min_version (sort -V puts smaller first).
+      if [ "$(printf '%s\n%s' "$min_version" "$ver" | sort -V | head -1)" = "$min_version" ]; then
+        versions+=("$ver")
+      fi
+    done < <(git tag -l "v${major}.*" | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+$" | sort -V)
+
+    # Exclude the current tag when running on a release tag push.
+    if [[ "${REF_NAME:-}" =~ ^v([0-9]+\.[0-9]+\.[0-9]+)$ ]]; then
+      current_tag="${BASH_REMATCH[1]}"
+      filtered=()
+      for v in "${versions[@]}"; do
+        [ "$v" != "$current_tag" ] && filtered+=("$v")
+      done
+      versions=("${filtered[@]}")
+    fi
+
+    if [ ${#versions[@]} -eq 0 ]; then
+      echo "No prior stable versions found for v${major}.x (>= $min_version). Skipping compat tests."
+      exit 0
+    fi
+
+    echo_header "Backwards compatibility e2e tests"
+    echo "Testing against ${#versions[@]} prior stable version(s): ${versions[*]}"
+
+    # Pre-populate the legacy contract cache on the host. Test containers run with --net=none, so the
+    # jest resolver's on-demand npm install would fail with EAI_AGAIN. Install here where we have network.
+    for ver in "${versions[@]}"; do
+      node yarn-project/end-to-end/src/install_legacy_contracts.cjs "$ver"
+    done
+
+    # Generate compat test commands for all versions and run them in parallel.
+    for ver in "${versions[@]}"; do
+      yarn-project/end-to-end/bootstrap.sh compat_test_cmds "$ver"
+    done | filter_test_cmds | parallelize
+    ;;
+
   ##########################################
   # ROLLUP UPGRADE DEPLOYMENT              #
   ##########################################

ci.sh

@@ -35,6 +35,7 @@ function print_usage {
   echo_cmd "network-teardown"      "Spin up an EC2 instance to teardown a network deployment."
   echo_cmd "network-tests-kind"    "Spin up an EC2 instance to run a KIND-based spartan test."
   echo_cmd "deploy-rollup-upgrade" "Spin up an EC2 instance to deploy a rollup upgrade."
+  echo_cmd "compat-e2e"            "Spin up an EC2 instance and run backwards compat e2e tests."
   echo_cmd "release"               "Spin up an EC2 instance and run bootstrap release."
   echo_cmd "shell-new"             "Spin up an EC2 instance, clone the repo, and drop into a shell."
   echo_cmd "shell"                 "Drop into a shell in the current running build instance container."
@@ -302,16 +303,42 @@ case "$cmd" in
     bootstrap_ec2 "./bootstrap.sh ci-deploy-rollup-upgrade $*"
     ;;
 
+  ##############################
+  # BACKWARDS COMPATIBILITY   #
+  ##############################
+  compat-e2e)
+    # Spin up an EC2 instance and run backwards compatibility e2e tests
+    # against contract artifacts from prior stable releases.
+    export CI_DASHBOARD="releases"
+    export JOB_ID="x-compat-e2e"
+    export AWS_SHUTDOWN_TIME=300
+    bootstrap_ec2 "./bootstrap.sh ci-compat-e2e"
+    ;;
+
   ############
   # RELEASES #
   ############
   release)
-    # Spin up ec2 instance and run the release flow.
+    # Spin up ec2 instance and run the release-tag verification build (no publish).
     export CI_DASHBOARD="releases"
     multi_job_run \
       'x-release amd64 ci-release' \
       'a-release arm64 ci-release'
     ;;
+  release-publish)
+    # Spin up ec2 instance and run the actual publish flow. Gated in ci3.yml on ci + ci-compat-e2e.
+    export CI_DASHBOARD="releases"
+    export DENOISE=1
+    export DENOISE_WIDTH=32
+    run() {
+      PARENT_LOG_ID=$RUN_ID JOB_ID=$1 INSTANCE_POSTFIX=$1 ARCH=$2 exec denoise "bootstrap_ec2 './bootstrap.sh ci-release-publish'"
+    }
+    export -f run
+
+    parallel --termseq 'TERM,10000' --tagstring '{= $_=~s/run (\w+).*/$1/; =}' --line-buffered --halt now,fail=1 ::: \
+      'run x-release-publish amd64' \
+      'run a-release-publish arm64' | DUP=1 cache_log "Release Publish CI run" $RUN_ID
+    ;;
 
   ##################
   # SHELL SESSIONS #

yarn-project/end-to-end/bootstrap.sh

@@ -246,6 +246,37 @@ function avm_check_circuit {
   avm_check_circuit_cmds | parallelize
 }
 
+# Generates e2e test commands using contract artifacts from a prior release version.
+# Only includes simple (jest-based) tests since compose/docker tests don't use the legacy jest resolver.
+# Excludes prover, block_building, and epochs tests (not relevant for contract artifact compat; epochs
+# tests are known-flaky and provide no additional backwards-compat coverage).
+function compat_test_cmds {
+  local version=${1:?version is required}
+  local run_test_script="yarn-project/end-to-end/scripts/run_test.sh"
+  local prefix="$hash:ISOLATE=1"
+  local compat_env="CONTRACT_ARTIFACTS_VERSION=$version"
+
+  local tests=(
+    src/e2e_!(prover|block_building|epochs)/*.test.ts
+    src/e2e_p2p/reqresp/*.test.ts
+    src/e2e_!(block_building|prover_*).test.ts
+  )
+  for test in "${tests[@]}"; do
+    local name=${test#*e2e_}
+    name=e2e_${name%.test.ts}
+
+    if [[ "$test" == *.parallel.test.ts ]]; then
+      while IFS= read -r test_name; do
+        local safe_test_name=$(echo "$test_name" | sed 's/ /_/g')
+        local full_name="compat_${version}_${name}_${safe_test_name}"
+        echo "$prefix:NAME=$full_name $compat_env $run_test_script simple $test \"$test_name\""
+      done < <(extract_test_names "$test")
+    else
+      echo "$prefix:NAME=compat_${version}_${name} $compat_env $run_test_script simple $test"
+    fi
+  done
+}
+
 case "$cmd" in
   "")
     build

yarn-project/end-to-end/src/install_legacy_contracts.cjs

@@ -0,0 +1,75 @@
+#!/usr/bin/env node
+// Installs pinned legacy @aztec/* contract-artifact packages into .legacy-contracts/<version>/.
+//
+// Called from two places:
+//   - bootstrap.sh ci-compat-e2e: pre-populates the cache on the host before hermetic test
+//     containers launch. The containers run with --net=none (see ci3/docker_isolate), so on-demand
+//     installs from inside them fail with EAI_AGAIN.
+//   - legacy-jest-resolver.cjs: on-demand install for local dev, where jest runs with network.
+//
+// Idempotent: no-op when all packages are already present.
+/* eslint-disable @typescript-eslint/no-require-imports */
+
+const path = require('path');
+const fs = require('fs');
+const { execFileSync } = require('child_process');
+
+const REDIRECTED = ['@aztec/noir-contracts.js', '@aztec/noir-test-contracts.js', '@aztec/accounts'];
+
+const e2eRoot = path.resolve(__dirname, '..');
+
+function cacheRoot(version) {
+  return path.join(e2eRoot, '.legacy-contracts', version);
+}
+
+function pkgJsonPath(version, pkg) {
+  return path.join(cacheRoot(version), 'node_modules', pkg, 'package.json');
+}
+
+function installLegacyContracts(version) {
+  if (!version) {
+    throw new Error('installLegacyContracts: version is required');
+  }
+  if (REDIRECTED.every(pkg => fs.existsSync(pkgJsonPath(version, pkg)))) {
+    return;
+  }
+
+  const cacheDir = cacheRoot(version);
+  fs.mkdirSync(cacheDir, { recursive: true });
+
+  // Seed a standalone package.json so `npm install --prefix` treats cacheRoot as its own project. Without this, npm
+  // walks up and finds the yarn-project workspace root, which breaks on `workspace:` protocol deps and risks
+  // clobbering the monorepo's node_modules.
+  const seed = path.join(cacheDir, 'package.json');
+  if (!fs.existsSync(seed)) {
+    fs.writeFileSync(seed, JSON.stringify({ name: 'legacy-contracts-cache', private: true }));
+  }
+
+  const specs = REDIRECTED.map(pkg => `${pkg}@${version}`);
+  process.stderr.write(`[legacy-contracts] installing ${specs.join(' ')} into ${cacheDir}\n`);
+  // --prefix: install into cacheRoot instead of cwd, so the cache is isolated from the monorepo.
+  // --no-save: don't write the installed packages back to the seeded package.json.
+  // --ignore-scripts: skip lifecycle scripts (preinstall/postinstall) of the legacy packages and their transitive
+  //   deps; we only want the files on disk, not to run any build steps.
+  // --legacy-peer-deps: tolerate peer-dependency mismatches between the pinned legacy @aztec/* graph and whatever
+  //   current versions npm would otherwise try to reconcile.
+  execFileSync(
+    'npm',
+    ['install', '--prefix', cacheDir, '--no-save', '--ignore-scripts', '--legacy-peer-deps', ...specs],
+    { stdio: 'inherit' },
+  );
+
+  // Verify versions on disk match the requested version.
+  for (const pkg of REDIRECTED) {
+    const onDisk = JSON.parse(fs.readFileSync(pkgJsonPath(version, pkg), 'utf8')).version;
+    if (onDisk !== version) {
+      throw new Error(`[legacy-contracts] ${pkg} on disk is ${onDisk}, expected ${version}`);
+    }
+  }
+}
+
+module.exports = { installLegacyContracts, REDIRECTED, cacheRoot };
+
+if (require.main === module) {
+  installLegacyContracts(process.argv[2]);
+}