Merge branch 'merge-train/fairies' into gj/embedded_wallet_improvements_and_regressions

Author: Thunkar

.github/ci3_labels_to_env.sh

@@ -37,6 +37,11 @@ function main {
     echo "NO_FAIL_FAST=1" >> $GITHUB_ENV
   fi
 
+  # Handle skip-compat-e2e label (escape hatch for backwards compat test failures on release PRs)
+  if has_label "ci-skip-compat-e2e"; then
+    echo "SKIP_COMPAT_E2E=1" >> $GITHUB_ENV
+  fi
+
   # Determine CI mode based on event, labels, and target branch
   local ci_mode
   if [ "${GITHUB_EVENT_NAME:-}" == "merge_group" ] || has_label "ci-merge-queue"; then

.github/workflows/ci3.yml

@@ -445,3 +445,108 @@ jobs:
           AWS_SHUTDOWN_TIME: 180
         run: |
           ./.github/ci3.sh network-tests-kind
+
+  # Backwards compatibility e2e tests.
+  # Runs e2e tests with contract artifacts from every prior stable release to validate
+  # that new client code works with old contract artifacts ("new pxe / old contracts").
+  # Blocking for stable/RC releases: ci-release-publish requires this job to pass before
+  # publishing. Observational for nightlies: runs, but continue-on-error keeps the workflow
+  # green and ci-release-publish's condition publishes nightlies regardless of the result.
+  # Escape hatch: ci-skip-compat-e2e label makes failures non-blocking on release PRs.
+  ci-compat-e2e:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    needs: [ci]
+    if: |
+      always()
+      && (needs.ci.result == 'success' || needs.ci.result == 'skipped')
+      && github.event.pull_request.head.repo.fork != true
+      && github.event.pull_request.draft == false
+      && (
+        (startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-commit.'))
+        || contains(github.event.pull_request.labels.*.name, 'ci-compat-e2e')
+        || contains(github.event.pull_request.labels.*.name, 'ci-release-pr')
+      )
+    # Non-blocking for nightlies and when ci-skip-compat-e2e escape hatch is applied.
+    continue-on-error: ${{ contains(github.ref_name, '-nightly.') || contains(github.event.pull_request.labels.*.name, 'ci-skip-compat-e2e') }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+
+      - name: Configure AWS credentials (OIDC)
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
+          aws-region: us-east-2
+          role-session-name: ci3-compat-e2e-${{ github.run_id }}
+          role-duration-seconds: 21600 # 6h – covers AWS_SHUTDOWN_TIME (300 min) + 60 min buffer
+
+      - name: Run Backwards Compatibility E2E Tests
+        timeout-minutes: 330
+        env:
+          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
+          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
+          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          CI3_INSTANCE_PROFILE_NAME: ${{ secrets.CI3_INSTANCE_PROFILE_NAME }}
+          CI3_SECURITY_GROUP_ID: ${{ secrets.CI3_SECURITY_GROUP_ID }}
+          RUN_ID: ${{ github.run_id }}
+          AWS_SHUTDOWN_TIME: 300
+        run: ./.github/ci3.sh compat-e2e
+
+  # Publishes the release (npm, Docker, GitHub release, aztec-up scripts, etc.).
+  # Gated on ci-compat-e2e: a compat regression blocks stable/RC publishing. Nightlies
+  # publish regardless — compat-e2e runs there observationally. Dev `-commit.` tags from
+  # the ci-release-pr flow never reach this job (they are not real releases).
+  ci-release-publish:
+    runs-on: ubuntu-latest
+    environment: master
+    permissions:
+      id-token: write
+      contents: read
+    needs: [ci, ci-compat-e2e]
+    if: |
+      startsWith(github.ref, 'refs/tags/v')
+      && !contains(github.ref_name, '-commit.')
+      && needs.ci.result == 'success'
+      && (
+        contains(github.ref_name, '-nightly.')
+        || needs.ci-compat-e2e.result == 'success'
+      )
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          ref: ${{ github.sha }}
+
+      - name: Configure AWS credentials (OIDC)
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
+          aws-region: us-east-2
+          role-session-name: ci3-release-publish-${{ github.run_id }}
+          role-duration-seconds: 21600
+
+      - name: Run Release Publish
+        env:
+          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
+          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
+          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
+          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          CI3_INSTANCE_PROFILE_NAME: ${{ secrets.CI3_INSTANCE_PROFILE_NAME }}
+          CI3_SECURITY_GROUP_ID: ${{ secrets.CI3_SECURITY_GROUP_ID }}
+          RUN_ID: ${{ github.run_id }}
+        run: ./.github/ci3.sh release-publish

bootstrap.sh

@@ -823,6 +823,17 @@ case "$cmd" in
   # RELEASES #
   ############
   "ci-release")
+    # Verification build for a release tag. Does NOT publish — publishing happens in
+    # ci-release-publish, gated on ci-compat-e2e so a compat regression blocks the release.
+    export CI=1
+    export USE_TEST_CACHE=1
+    if ! semver check $REF_NAME; then
+      exit 1
+    fi
+    build
+    ;;
+  "ci-release-publish")
+    # Actual publish step. `build` cache-hits against ci-release's build of the same commit.
     export CI=1
     export USE_TEST_CACHE=1
     if ! semver check $REF_NAME; then
@@ -902,6 +913,82 @@ case "$cmd" in
     build
     yarn-project/end-to-end/bootstrap.sh avm_check_circuit
     ;;
+  #############################################
+  # BACKWARDS COMPATIBILITY E2E TESTS         #
+  #############################################
+  "ci-compat-e2e")
+    # Runs e2e tests with contract artifacts from every prior stable release since 4.2.0 (version where we committed to
+    # backwards compatibility). This Validates that old contract artifacts work on current release.
+    export CI=1
+    export USE_TEST_CACHE=0
+    export CI_FULL=0
+    export NO_FAIL_FAST=1
+
+    build
+
+    # TODO: bump when v5 commits to backwards-compatible contract artifacts.
+    #   compat_major:       major version that has compat guarantees today.
+    #   compat_min_version: earliest stable tag of that major to test against
+    #                       (artifacts before this are incompatible due to oracle interface changes).
+    compat_major="4"
+    compat_min_version="4.2.0"
+
+    # Get current major version.
+    current_version=$(jq -r '."."' .release-please-manifest.json)
+    major=$(semver major "$current_version")
+    if [ "$major" != "$compat_major" ]; then
+      echo "Compat e2e tests only apply to v${compat_major}. Current major: v${major}. Skipping."
+      exit 0
+    fi
+    min_version="$compat_min_version"
+
+    # Fetch tags (EC2 clone may not have them). Fail loud: a silent fetch failure plus an empty
+    # tag list would publish a real release with zero compat coverage.
+    if ! git fetch origin 'refs/tags/v*:refs/tags/v*'; then
+      echo "ERROR: failed to fetch release tags." >&2
+      exit 1
+    fi
+
+    # Discover stable tags for this major version (no prerelease suffixes).
+    versions=()
+    while IFS= read -r tag; do
+      ver=${tag#v}
+      # Include only versions >= min_version (sort -V puts smaller first).
+      if [ "$(printf '%s\n%s' "$min_version" "$ver" | sort -V | head -1)" = "$min_version" ]; then
+        versions+=("$ver")
+      fi
+    done < <(git tag -l "v${major}.*" | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+$" | sort -V)
+
+    # Exclude the current tag when running on a release tag push.
+    if [[ "${REF_NAME:-}" =~ ^v([0-9]+\.[0-9]+\.[0-9]+)$ ]]; then
+      current_tag="${BASH_REMATCH[1]}"
+      filtered=()
+      for v in "${versions[@]}"; do
+        [ "$v" != "$current_tag" ] && filtered+=("$v")
+      done
+      versions=("${filtered[@]}")
+    fi
+
+    if [ ${#versions[@]} -eq 0 ]; then
+      echo "No prior stable versions found for v${major}.x (>= $min_version). Skipping compat tests."
+      exit 0
+    fi
+
+    echo_header "Backwards compatibility e2e tests"
+    echo "Testing against ${#versions[@]} prior stable version(s): ${versions[*]}"
+
+    # Pre-populate the legacy contract cache on the host. Test containers run with --net=none, so the
+    # jest resolver's on-demand npm install would fail with EAI_AGAIN. Install here where we have network.
+    for ver in "${versions[@]}"; do
+      node yarn-project/end-to-end/src/install_legacy_contracts.cjs "$ver"
+    done
+
+    # Generate compat test commands for all versions and run them in parallel.
+    for ver in "${versions[@]}"; do
+      yarn-project/end-to-end/bootstrap.sh compat_test_cmds "$ver"
+    done | filter_test_cmds | parallelize
+    ;;
+
   ##########################################
   # ROLLUP UPGRADE DEPLOYMENT              #
   ##########################################

ci.sh

@@ -35,6 +35,7 @@ function print_usage {
   echo_cmd "network-teardown"      "Spin up an EC2 instance to teardown a network deployment."
   echo_cmd "network-tests-kind"    "Spin up an EC2 instance to run a KIND-based spartan test."
   echo_cmd "deploy-rollup-upgrade" "Spin up an EC2 instance to deploy a rollup upgrade."
+  echo_cmd "compat-e2e"            "Spin up an EC2 instance and run backwards compat e2e tests."
   echo_cmd "release"               "Spin up an EC2 instance and run bootstrap release."
   echo_cmd "shell-new"             "Spin up an EC2 instance, clone the repo, and drop into a shell."
   echo_cmd "shell"                 "Drop into a shell in the current running build instance container."
@@ -302,16 +303,42 @@ case "$cmd" in
     bootstrap_ec2 "./bootstrap.sh ci-deploy-rollup-upgrade $*"
     ;;
 
+  ##############################
+  # BACKWARDS COMPATIBILITY   #
+  ##############################
+  compat-e2e)
+    # Spin up an EC2 instance and run backwards compatibility e2e tests
+    # against contract artifacts from prior stable releases.
+    export CI_DASHBOARD="releases"
+    export JOB_ID="x-compat-e2e"
+    export AWS_SHUTDOWN_TIME=300
+    bootstrap_ec2 "./bootstrap.sh ci-compat-e2e"
+    ;;
+
   ############
   # RELEASES #
   ############
   release)
-    # Spin up ec2 instance and run the release flow.
+    # Spin up ec2 instance and run the release-tag verification build (no publish).
     export CI_DASHBOARD="releases"
     multi_job_run \
       'x-release amd64 ci-release' \
       'a-release arm64 ci-release'
     ;;
+  release-publish)
+    # Spin up ec2 instance and run the actual publish flow. Gated in ci3.yml on ci + ci-compat-e2e.
+    export CI_DASHBOARD="releases"
+    export DENOISE=1
+    export DENOISE_WIDTH=32
+    run() {
+      PARENT_LOG_ID=$RUN_ID JOB_ID=$1 INSTANCE_POSTFIX=$1 ARCH=$2 exec denoise "bootstrap_ec2 './bootstrap.sh ci-release-publish'"
+    }
+    export -f run
+
+    parallel --termseq 'TERM,10000' --tagstring '{= $_=~s/run (\w+).*/$1/; =}' --line-buffered --halt now,fail=1 ::: \
+      'run x-release-publish amd64' \
+      'run a-release-publish arm64' | DUP=1 cache_log "Release Publish CI run" $RUN_ID
+    ;;
 
   ##################
   # SHELL SESSIONS #

noir-projects/aztec-nr/aztec/src/contract_self/contract_self_private.nr

@@ -48,7 +48,8 @@ use crate::protocol::{address::AztecAddress, traits::{Deserialize, Serialize}};
 /// - `CallSelfStatic`: Macro-generated type for calling contract's own view functions
 /// - `EnqueueSelfStatic`: Macro-generated type for enqueuing calls to the contract's own view functions
 /// - `CallInternal`: Macro-generated type for calling internal functions
-pub struct ContractSelfPrivate<Storage, CallSelf, EnqueueSelf, CallSelfStatic, EnqueueSelfStatic, CallInternal> {
+/// - `CallSelfUtility`: Macro-generated type for calling the contract's own utility functions
+pub struct ContractSelfPrivate<Storage, CallSelf, EnqueueSelf, CallSelfStatic, EnqueueSelfStatic, CallInternal, CallSelfUtility> {
     /// The address of this contract
     pub address: AztecAddress,
 
@@ -110,17 +111,17 @@ pub struct ContractSelfPrivate<Storage, CallSelf, EnqueueSelf, CallSelfStatic, E
     /// ```
     pub internal: CallInternal,
 
-    /// Gateway for calling utility functions from this private context.
+    /// A struct that allows for ergonomic calling of utility functions from this private context.
     ///
     /// Example API:
     /// ```noir
     /// // Safety: result is unconstrained
     /// unsafe { self.utility.call(MyContract::at(address).my_utility_function(args)) }
     /// ```
-    pub utility: PrivateUtility,
+    pub utility: PrivateUtilityCalls<CallSelfUtility>,
 }
 
-impl<Storage, CallSelf, EnqueueSelf, CallSelfStatic, EnqueueSelfStatic, CallInternal> ContractSelfPrivate<Storage, CallSelf, EnqueueSelf, CallSelfStatic, EnqueueSelfStatic, CallInternal> {
+impl<Storage, CallSelf, EnqueueSelf, CallSelfStatic, EnqueueSelfStatic, CallInternal, CallSelfUtility> ContractSelfPrivate<Storage, CallSelf, EnqueueSelf, CallSelfStatic, EnqueueSelfStatic, CallInternal, CallSelfUtility> {
     /// Creates a new `ContractSelfPrivate` instance for a private function.
     ///
     /// This constructor is called automatically by the macro system and should not be called directly.
@@ -132,6 +133,7 @@ impl<Storage, CallSelf, EnqueueSelf, CallSelfStatic, EnqueueSelfStatic, CallInte
         call_self_static: CallSelfStatic,
         enqueue_self_static: EnqueueSelfStatic,
         internal: CallInternal,
+        utility: PrivateUtilityCalls<CallSelfUtility>,
     ) -> Self {
         Self {
             context,
@@ -142,7 +144,7 @@ impl<Storage, CallSelf, EnqueueSelf, CallSelfStatic, EnqueueSelfStatic, CallInte
             call_self_static,
             enqueue_self_static,
             internal,
-            utility: PrivateUtility {},
+            utility,
         }
     }
 
@@ -407,16 +409,31 @@ impl<Storage, CallSelf, EnqueueSelf, CallSelfStatic, EnqueueSelfStatic, CallInte
     }
 }
 
-/// Gateway for calling utility functions from a private context.
+/// A struct that allows for ergonomic calling of utility functions from a private context.
 ///
 /// Accessible via `self.utility` in private functions. Results are not part of any circuit
 /// proof; use them to inform logic, not as inputs to constrained assertions.
-// Implemented as a separate empty struct because Noir does not allow passing `&mut PrivateContext`
-// (held by `ContractSelfPrivate`) into unconstrained code. This struct holds no mutable references,
-// so it can be passed freely into `unconstrained` functions.
-pub struct PrivateUtility {}
+///
+/// ## Type Parameters
+///
+/// - `CallSelf`: Macro-generated type for calling the contract's own utility functions (same type
+/// as `CallSelf` in \[`ContractSelfUtility`\])
+// Implemented as a separate struct (rather than inlined in ContractSelfPrivate) because Noir does
+// not allow passing `&mut PrivateContext` (held by `ContractSelfPrivate`) into unconstrained code.
+// This struct holds no mutable references, so it can be passed freely into `unconstrained`
+// functions.
+pub struct PrivateUtilityCalls<CallSelf> {
+    /// Provides type-safe methods for calling this contract's own utility functions.
+    ///
+    /// Example API:
+    /// ```noir
+    /// // Safety: result is unconstrained
+    /// unsafe { self.utility.call_self.some_utility_function(args) }
+    /// ```
+    pub call_self: CallSelf,
+}
 
-impl PrivateUtility {
+impl<CallSelf> PrivateUtilityCalls<CallSelf> {
     /// Makes a utility contract call from a private function.
     ///
     /// Note: only same-contract utility calls are currently supported. See TODO(F-29).

noir-projects/aztec-nr/aztec/src/contract_self/contract_self_utility.nr

@@ -13,7 +13,9 @@ use crate::protocol::{address::AztecAddress, traits::Deserialize};
 ///
 /// - `Storage`: The contract's storage struct (defined with [`storage`](crate::macros::storage::storage), or `()` if
 /// the contract has no storage
-pub struct ContractSelfUtility<Storage> {
+/// - `CallSelf`: Macro-generated type for calling the contract's own utility functions (same type
+/// as `CallSelf` in \[`PrivateUtilityCalls`\])
+pub struct ContractSelfUtility<Storage, CallSelf> {
     /// The address of this contract
     pub address: AztecAddress,
 
@@ -34,14 +36,22 @@ pub struct ContractSelfUtility<Storage> {
 
     /// The utility execution context.
     pub context: UtilityContext,
+
+    /// Provides type-safe methods for calling this contract's own utility functions.
+    ///
+    /// Example API:
+    /// ```noir
+    /// self.call_self.some_utility_function(args)
+    /// ```
+    pub call_self: CallSelf,
 }
 
-impl<Storage> ContractSelfUtility<Storage> {
+impl<Storage, CallSelf> ContractSelfUtility<Storage, CallSelf> {
     /// Creates a new `ContractSelfUtility` instance for a utility function.
     ///
     /// This constructor is called automatically by the macro system and should not be called directly.
-    pub fn new(context: UtilityContext, storage: Storage) -> Self {
-        Self { context, storage, address: context.this_address() }
+    pub fn new(context: UtilityContext, storage: Storage, call_self: CallSelf) -> Self {
+        Self { context, storage, address: context.this_address(), call_self }
     }
 
     /// Makes a utility contract call from another utility function.