Merge branch 'next' into claudebox/fix-nightly-bb-debug-build

Author: iakovenkos

.github/ci3_labels_to_env.sh

@@ -37,6 +37,11 @@ function main {
     echo "NO_FAIL_FAST=1" >> $GITHUB_ENV
   fi
 
+  # Handle skip-compat-e2e label (escape hatch for backwards compat test failures on release PRs)
+  if has_label "ci-skip-compat-e2e"; then
+    echo "SKIP_COMPAT_E2E=1" >> $GITHUB_ENV
+  fi
+
   # Determine CI mode based on event, labels, and target branch
   local ci_mode
   if [ "${GITHUB_EVENT_NAME:-}" == "merge_group" ] || has_label "ci-merge-queue"; then

.github/workflows/ci3.yml

@@ -445,3 +445,108 @@ jobs:
           AWS_SHUTDOWN_TIME: 180
         run: |
           ./.github/ci3.sh network-tests-kind
+
+  # Backwards compatibility e2e tests.
+  # Runs e2e tests with contract artifacts from every prior stable release to validate
+  # that new client code works with old contract artifacts ("new pxe / old contracts").
+  # Blocking for stable/RC releases: ci-release-publish requires this job to pass before
+  # publishing. Observational for nightlies: runs, but continue-on-error keeps the workflow
+  # green and ci-release-publish's condition publishes nightlies regardless of the result.
+  # Escape hatch: ci-skip-compat-e2e label makes failures non-blocking on release PRs.
+  ci-compat-e2e:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    needs: [ci]
+    if: |
+      always()
+      && (needs.ci.result == 'success' || needs.ci.result == 'skipped')
+      && github.event.pull_request.head.repo.fork != true
+      && github.event.pull_request.draft == false
+      && (
+        (startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-commit.'))
+        || contains(github.event.pull_request.labels.*.name, 'ci-compat-e2e')
+        || contains(github.event.pull_request.labels.*.name, 'ci-release-pr')
+      )
+    # Non-blocking for nightlies and when ci-skip-compat-e2e escape hatch is applied.
+    continue-on-error: ${{ contains(github.ref_name, '-nightly.') || contains(github.event.pull_request.labels.*.name, 'ci-skip-compat-e2e') }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+
+      - name: Configure AWS credentials (OIDC)
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
+          aws-region: us-east-2
+          role-session-name: ci3-compat-e2e-${{ github.run_id }}
+          role-duration-seconds: 21600 # 6h – covers AWS_SHUTDOWN_TIME (300 min) + 60 min buffer
+
+      - name: Run Backwards Compatibility E2E Tests
+        timeout-minutes: 330
+        env:
+          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
+          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
+          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          CI3_INSTANCE_PROFILE_NAME: ${{ secrets.CI3_INSTANCE_PROFILE_NAME }}
+          CI3_SECURITY_GROUP_ID: ${{ secrets.CI3_SECURITY_GROUP_ID }}
+          RUN_ID: ${{ github.run_id }}
+          AWS_SHUTDOWN_TIME: 300
+        run: ./.github/ci3.sh compat-e2e
+
+  # Publishes the release (npm, Docker, GitHub release, aztec-up scripts, etc.).
+  # Gated on ci-compat-e2e: a compat regression blocks stable/RC publishing. Nightlies
+  # publish regardless — compat-e2e runs there observationally. Dev `-commit.` tags from
+  # the ci-release-pr flow never reach this job (they are not real releases).
+  ci-release-publish:
+    runs-on: ubuntu-latest
+    environment: master
+    permissions:
+      id-token: write
+      contents: read
+    needs: [ci, ci-compat-e2e]
+    if: |
+      startsWith(github.ref, 'refs/tags/v')
+      && !contains(github.ref_name, '-commit.')
+      && needs.ci.result == 'success'
+      && (
+        contains(github.ref_name, '-nightly.')
+        || needs.ci-compat-e2e.result == 'success'
+      )
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          ref: ${{ github.sha }}
+
+      - name: Configure AWS credentials (OIDC)
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
+          aws-region: us-east-2
+          role-session-name: ci3-release-publish-${{ github.run_id }}
+          role-duration-seconds: 21600
+
+      - name: Run Release Publish
+        env:
+          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
+          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
+          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
+          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          CI3_INSTANCE_PROFILE_NAME: ${{ secrets.CI3_INSTANCE_PROFILE_NAME }}
+          CI3_SECURITY_GROUP_ID: ${{ secrets.CI3_SECURITY_GROUP_ID }}
+          RUN_ID: ${{ github.run_id }}
+        run: ./.github/ci3.sh release-publish

.github/workflows/metrics-deploy.yml

@@ -37,6 +37,11 @@ on:
         required: true
         type: string
         default: "grafana-dashboard-password"
+      slack_alert_mention_user_ids:
+        description: Optional Terraform list of Slack user IDs to mention on Grafana alert notifications
+        required: false
+        type: string
+        default: '["U0AHB6VR8N5"]'
     secrets:
       GCP_SA_KEY:
         required: true
@@ -70,6 +75,10 @@ on:
         description: The name of the secret which holds the Grafana dashboard password
         required: true
         default: "grafana-dashboard-password"
+      slack_alert_mention_user_ids:
+        description: Optional Terraform list of Slack user IDs to mention on Grafana alert notifications
+        required: false
+        default: '["U0AHB6VR8N5"]'
 
 jobs:
   metrics_deployment:
@@ -96,6 +105,7 @@ jobs:
       SLACK_WEBHOOK_NEXT_NET_SECRET_NAME: slack-webhook-next-net-url
       SLACK_WEBHOOK_TESTNET_SECRET_NAME: slack-webhook-testnet-url
       SLACK_WEBHOOK_MAINNET_SECRET_NAME: slack-webhook-mainnet-url
+      TF_VAR_SLACK_ALERT_MENTION_USER_IDS: ${{ inputs.slack_alert_mention_user_ids }}
 
     steps:
       - name: Checkout code

barretenberg/bbup/bbup

@@ -50,8 +50,8 @@ get_bb_version_for_noir() {
 
     local lookup_url="https://raw.githubusercontent.com/AztecProtocol/aztec-packages/next/barretenberg/bbup/bb-versions.json"
 
-    # Extract BB version from install script
-    local bb_version=$(curl --fail -s "$lookup_url" | jq -r --arg version "$resolved_version" '.[$version]')
+    # bb-versions.json is a flat "<noir-version>": "<bb-version>" map with one entry per line.
+    local bb_version=$(curl --fail -s "$lookup_url" | grep -F "\"$resolved_version\":" | head -1 | cut -d'"' -f4)
     echo "$bb_version"
 }
 

barretenberg/cpp/scripts/test_chonk_standalone_vks_havent_changed.sh

@@ -21,7 +21,7 @@ script_path="$root/barretenberg/cpp/scripts/test_chonk_standalone_vks_havent_cha
 # - Generate a hash for versioning: sha256sum bb-chonk-inputs.tar.gz
 # - Upload the compressed results: aws s3 cp bb-chonk-inputs.tar.gz s3://aztec-ci-artifacts/protocol/bb-chonk-inputs-[hash(0:8)].tar.gz
 # Note: In case of the "Test suite failed to run ... Unexpected token 'with' " error, need to run: docker pull aztecprotocol/build:3.0
-pinned_short_hash="ae9c2d06"
+pinned_short_hash="c09aeb0c"
 pinned_chonk_inputs_url="https://aztec-ci-artifacts.s3.us-east-2.amazonaws.com/protocol/bb-chonk-inputs-${pinned_short_hash}.tar.gz"
 
 function update_pinned_hash_in_script {

barretenberg/cpp/src/barretenberg/bbapi/bbapi_ecc.cpp

@@ -99,6 +99,12 @@ Bn254G2Mul::Response Bn254G2Mul::execute(BBApiRequest& request) &&
     if (!point.on_curve()) {
         BBAPI_ERROR(request, "Input point must be on the curve");
     }
+    // BN254 G2 has cofactor h2 ≈ 2^254. An on-curve point may lie in a cofactor subgroup of order
+    // dividing h2 rather than the prime-order subgroup; we do not want to allow such points
+    // as inputs to bbapi.
+    if (!point.is_in_prime_subgroup()) {
+        BBAPI_ERROR(request, "Input point must lie in the prime-order subgroup");
+    }
     auto result = point * scalar;
     if (!result.on_curve()) {
         BBAPI_ERROR(request, "Output point must be on the curve");

barretenberg/cpp/src/barretenberg/bbapi/bbapi_srs.cpp

@@ -50,8 +50,15 @@ SrsInitSrs::Response SrsInitSrs::execute(BB_UNUSED BBApiRequest& request) &&
                        std::to_string(bytes_per_point));
     }
 
-    // Parse G2 point from buffer (128 bytes)
+    // Parse G2 point from buffer (128 bytes). `serialize_from_buffer` validates that the bytes
+    // decode to a curve point but does NOT enforce subgroup membership. BN254 G2 has a non-trivial
+    // cofactor (h2 ≈ 2^254), so a curve point may lie in a small cofactor subgroup of order
+    // dividing h2 rather than the prime-order subgroup of order r. Reject anything outside
+    // the prime-order subgroup before it reaches the SRS factory.
     auto g2_point_elem = from_buffer<g2::affine_element>(g2_point.data());
+    if (!g2_point_elem.is_in_prime_subgroup()) {
+        throw_or_abort("SrsInitSrs: g2_point is not in the BN254 G2 prime-order subgroup");
+    }
 
     // Initialize BN254 SRS
     bb::srs::init_bn254_mem_crs_factory(g1_points, g2_point_elem);

barretenberg/cpp/src/barretenberg/boomerang_value_detection/graph_description_goblin.test.cpp

@@ -45,14 +45,16 @@ class BoomerangGoblinRecursiveVerifierTests : public testing::Test {
     {
         Goblin goblin;
         GoblinMockCircuits::construct_and_merge_mock_circuits(goblin, 5);
+        goblin.op_queue->construct_zk_columns();
 
         // Merge the ecc ops from the newly constructed circuit
         auto goblin_proof = goblin.prove();
         // Subtable values and commitments - needed for (Recursive)MergeVerifier
         MergeCommitments merge_commitments;
         auto t_current = goblin.op_queue->construct_current_ultra_ops_subtable_columns();
-        auto T_prev = goblin.op_queue->construct_previous_ultra_ops_table_columns();
-        CommitmentKey<curve::BN254> pcs_commitment_key(goblin.op_queue->get_ultra_ops_table_num_rows());
+        auto T_prev = goblin.op_queue->construct_table_columns_up_to_tail();
+        CommitmentKey<curve::BN254> pcs_commitment_key(goblin.op_queue->get_ultra_ops_table_num_rows() +
+                                                       UltraEccOpsTable::ZK_ULTRA_OPS);
         for (size_t idx = 0; idx < MegaFlavor::NUM_WIRES; idx++) {
             merge_commitments.t_commitments[idx] = pcs_commitment_key.commit(t_current[idx]);
             merge_commitments.T_prev_commitments[idx] = pcs_commitment_key.commit(T_prev[idx]);
@@ -86,7 +88,7 @@ TEST_F(BoomerangGoblinRecursiveVerifierTests, graph_description_basic)
 
     auto transcript = std::make_shared<GoblinRecursiveVerifier::Transcript>();
     GoblinStdlibProof stdlib_proof(builder, proof);
-    GoblinRecursiveVerifier verifier{ transcript, stdlib_proof, recursive_merge_commitments, MergeSettings::APPEND };
+    GoblinRecursiveVerifier verifier{ transcript, stdlib_proof, recursive_merge_commitments };
     GoblinRecursiveVerifier::ReductionResult output = verifier.reduce_to_pairing_check_and_ipa_opening();
 
     // Aggregate merge + translator pairing points

barretenberg/cpp/src/barretenberg/boomerang_value_detection/graph_description_goblin_avm.test.cpp

@@ -54,17 +54,14 @@ class BoomerangGoblinAvmRecursiveVerifierTests : public testing::Test {
         GoblinAvm goblin(inner_builder);
         MockCircuits::construct_arithmetic_circuit(inner_builder);
 
-        // Build a MegaAvm prover instance to get ecc_op_wire commitments matching the real flow.
-        auto mega_avm_instance = std::make_shared<ProverInstance_<MegaAvmFlavor>>(inner_builder);
-        CommitmentKey<curve::BN254> pcs_commitment_key(mega_avm_instance->dyadic_size());
-
         auto goblin_proof = goblin.prove();
 
-        // Commit to ecc_op_wire polynomials from the MegaAvm prover instance
+        // Commit to op_queue columns.
         TableCommitments table_commitments;
-        size_t idx = 0;
-        for (auto& wire : mega_avm_instance->polynomials.get_ecc_op_wires()) {
-            table_commitments[idx++] = pcs_commitment_key.commit(wire);
+        auto ultra_ops_table_columns = goblin.op_queue->construct_ultra_ops_table_columns(/*include_zk_ops=*/false);
+        CommitmentKey<curve::BN254> pcs_commitment_key(goblin.op_queue->get_ultra_ops_table_num_rows());
+        for (size_t idx = 0; idx < MegaFlavor::NUM_WIRES; idx++) {
+            table_commitments[idx] = pcs_commitment_key.commit(ultra_ops_table_columns[idx]);
         }
 
         RecursiveTableCommitments recursive_table_commitments;

barretenberg/cpp/src/barretenberg/boomerang_value_detection/graph_description_merge_recursive_verifier.test.cpp

@@ -47,22 +47,37 @@ template <class RecursiveBuilder> class BoomerangRecursiveMergeVerifierTest : pu
         EXPECT_EQ(result.second.size(), 0);
     }
 
-    static void prove_and_verify_merge(const std::shared_ptr<ECCOpQueue>& op_queue,
-                                       const MergeSettings settings = MergeSettings::PREPEND,
-                                       const bool run_analyzer = false)
+    static std::shared_ptr<ECCOpQueue> construct_final_merge_op_queue(const size_t num_subtables_up_to_tail)
+    {
+        auto op_queue = std::make_shared<ECCOpQueue>();
+
+        for (size_t idx = 0; idx < num_subtables_up_to_tail; ++idx) {
+            InnerBuilder circuit{ op_queue };
+            GoblinMockCircuits::construct_simple_circuit(circuit);
+            op_queue->merge();
+        }
+
+        op_queue->construct_zk_columns();
+
+        InnerBuilder hiding_circuit{ op_queue };
+        GoblinMockCircuits::construct_simple_circuit(hiding_circuit);
+        return op_queue;
+    }
+
+    static void prove_and_verify_merge(const std::shared_ptr<ECCOpQueue>& op_queue, const bool run_analyzer = false)
 
     {
         RecursiveBuilder outer_circuit;
 
         auto prover_transcript = std::make_shared<NativeTranscript>();
-        MergeProver merge_prover{ op_queue, prover_transcript, settings };
+        MergeProver merge_prover{ op_queue, prover_transcript };
         auto merge_proof = merge_prover.construct_proof();
 
         // Subtable values and commitments - needed for (Recursive)MergeVerifier
         MergeCommitments merge_commitments;
         RecursiveMergeCommitments recursive_merge_commitments;
         auto t_current = op_queue->construct_current_ultra_ops_subtable_columns();
-        auto T_prev = op_queue->construct_previous_ultra_ops_table_columns();
+        auto T_prev = op_queue->construct_table_columns_up_to_tail();
         for (size_t idx = 0; idx < InnerFlavor::NUM_WIRES; idx++) {
             merge_commitments.t_commitments[idx] = merge_prover.pcs_commitment_key.commit(t_current[idx]);
             merge_commitments.T_prev_commitments[idx] = merge_prover.pcs_commitment_key.commit(T_prev[idx]);
@@ -78,7 +93,7 @@ template <class RecursiveBuilder> class BoomerangRecursiveMergeVerifierTest : pu
 
         // Create a recursive merge verification circuit for the merge proof
         auto merge_transcript = std::make_shared<StdlibTranscript<RecursiveBuilder>>();
-        RecursiveMergeVerifier verifier{ settings, merge_transcript };
+        RecursiveMergeVerifier verifier{ merge_transcript };
         const stdlib::Proof<RecursiveBuilder> stdlib_merge_proof(outer_circuit, merge_proof);
         auto [pairing_points, merged_commitments, reduction_succeeded] =
             verifier.reduce_to_pairing_check(stdlib_merge_proof, recursive_merge_commitments);
@@ -96,53 +111,20 @@ template <class RecursiveBuilder> class BoomerangRecursiveMergeVerifierTest : pu
         }
     }
 
-    static void test_recursive_merge_verification_prepend()
-    {
-        auto op_queue = std::make_shared<ECCOpQueue>();
-
-        InnerBuilder circuit{ op_queue };
-        GoblinMockCircuits::construct_simple_circuit(circuit);
-        prove_and_verify_merge(op_queue);
-
-        InnerBuilder circuit2{ op_queue };
-        GoblinMockCircuits::construct_simple_circuit(circuit2);
-        prove_and_verify_merge(op_queue);
-
-        InnerBuilder circuit3{ op_queue };
-        GoblinMockCircuits::construct_simple_circuit(circuit3);
-        prove_and_verify_merge(op_queue, MergeSettings::PREPEND, true);
-    }
-
-    static void test_recursive_merge_verification_append()
+    static void test_recursive_merge_verification()
     {
-        auto op_queue = std::make_shared<ECCOpQueue>();
-
-        InnerBuilder circuit{ op_queue };
-        GoblinMockCircuits::construct_simple_circuit(circuit);
-        prove_and_verify_merge(op_queue);
-
-        InnerBuilder circuit2{ op_queue };
-        GoblinMockCircuits::construct_simple_circuit(circuit2);
-        prove_and_verify_merge(op_queue);
-
-        InnerBuilder circuit3{ op_queue };
-        GoblinMockCircuits::construct_simple_circuit(circuit3);
-        prove_and_verify_merge(op_queue, MergeSettings::APPEND, true);
+        auto op_queue = construct_final_merge_op_queue(/*num_subtables_up_to_tail=*/3);
+        prove_and_verify_merge(op_queue, /*run_analyzer=*/true);
     }
 };
 
 using Builder = testing::Types<MegaCircuitBuilder>;
 
 TYPED_TEST_SUITE(BoomerangRecursiveMergeVerifierTest, Builder);
 
-TYPED_TEST(BoomerangRecursiveMergeVerifierTest, RecursiveVerificationPrepend)
-{
-    TestFixture::test_recursive_merge_verification_prepend();
-};
-
-TYPED_TEST(BoomerangRecursiveMergeVerifierTest, RecursiveVerificationAppend)
+TYPED_TEST(BoomerangRecursiveMergeVerifierTest, RecursiveMergeVerification)
 {
-    TestFixture::test_recursive_merge_verification_append();
+    TestFixture::test_recursive_merge_verification();
 };
 
 } // namespace bb::stdlib::recursion::goblin