feat: barretenberg merge-train (#15292)

fix: merge train shorter body (#15331)
fix: merge recreate automation (#15332)
fix: pr-body for merge-train
fix: merge-train pr body workflow
fix: ci3.yml merge-train concurrency
Update merge-train-update-pr-body.yml
chore: make bb tests have BB_VERBOSE=1 (#15328)
chore: simplify trace poly population methods (#15298)
chore: remove deprecated create_circuit method (#15296)
chore: construct thread ranges as list of active ranges (#15285)

---------

Co-authored-by: Jonathan Hao <jonathan@aztec-labs.com>
Co-authored-by: AztecBot <tech@aztecprotocol.com>
Co-authored-by: ledwards2225 <98505400+ledwards2225@users.noreply.github.com>

Author: 4 people

.github/workflows/ci3.yml

@@ -75,7 +75,7 @@ jobs:
       # Allow full concurrency for merge-train PRs, one-run-per-branch for everything else.
       - name: Set Instance Postfix for merge-train
         if: startsWith(github.event.pull_request.head.ref, 'merge-train/')
-        run: echo "INSTANCE_POSTFIX=merge-train-${{ github.sha }}" >> $GITHUB_ENV
+        run: echo "INSTANCE_POSTFIX=${{ github.event.pull_request.commits }}" >> $GITHUB_ENV
 
       - name: Setup
         run: |

…workflows/merge-train-recreate-queue.yml …ithub/workflows/merge-train-recreate.yml.github/workflows/merge-train-recreate-queue.yml renamed to .github/workflows/merge-train-recreate.yml .github/workflows/merge-train-recreate-queue.yml renamed to .github/workflows/merge-train-recreate.yml

@@ -40,6 +40,10 @@ jobs:
           NEW_BRANCH="merge-queue/${MERGE_TRAIN_BRANCH}/${TIMESTAMP}"
           git checkout -b "$NEW_BRANCH"
 
+          # Merge the base branch (typically 'next') with theirs strategy
+          BASE_BRANCH="${MERGE_TRAIN_BRANCH#merge-train/}"
+          git merge -X theirs "origin/$BASE_BRANCH"
+
           # Seed it with an empty commit
           git commit --allow-empty -m "[empty] Start merge-train. Choo choo."
 

.github/workflows/merge-train-update-pr-body.yml

@@ -3,7 +3,7 @@ name: Merge-Train PR Body
 on:
   push:
     branches:
-      - 'merge-train/**'   # this time it *is* the branch that was pushed
+      - 'merge-train/**'
 
 jobs:
   update-pr-body:
@@ -37,13 +37,16 @@ jobs:
         id: commits
         run: |
           PR_NUMBER=${{ steps.pr.outputs.number }}
-          BASE_SHA=$(gh pr view $PR_NUMBER --json baseRef --jq .baseRef.target.oid)
-          HEAD_SHA=$GITHUB_SHA
+          export GH_TOKEN="${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}"
+          # Obtain the base-branch and head-branch tip commits
+          read BASE_SHA HEAD_SHA < <(
+            gh pr view "$PR_NUMBER" \
+              --json baseRefOid,headRefOid \
+              --jq '.baseRefOid + " " + .headRefOid'
+          )
           echo "commit_list<<EOF" >> "$GITHUB_OUTPUT"
           git log --oneline --no-merges "$BASE_SHA..$HEAD_SHA" --pretty=format:"%s" |
-            grep -v "^\[empty\]" |
-            grep -v "^Start merge-train" |
-            sed 's/^/- /' >> "$GITHUB_OUTPUT"
+            grep -v "^\[empty\]" >> "$GITHUB_OUTPUT"
           echo "" >> "$GITHUB_OUTPUT"
           echo "EOF" >> "$GITHUB_OUTPUT"
 
@@ -54,7 +57,7 @@ jobs:
           script: |
             const number = ${{ steps.pr.outputs.number }};
             const commits = `${{ steps.commits.outputs.commit_list }}`.trim();
-            const newBody = `Merge train with:\n${commits || '*No commits yet*'}`;
+            const newBody = `${commits || '*No commits yet*'}`;
             await github.rest.pulls.update({
               owner: context.repo.owner,
               repo: context.repo.repo,

barretenberg/cpp/scripts/run_test.sh

@@ -12,5 +12,6 @@ cd $(scripts/native-preset-build-dir)
 
 export GTEST_COLOR=1
 export HARDWARE_CONCURRENCY=${CPUS:-8}
+export BB_VERBOSE=1
 
 exec ./bin/$1 --gtest_filter=$2

barretenberg/cpp/src/barretenberg/circuit_checker/ultra_circuit_checker.cpp

@@ -279,10 +279,11 @@ void UltraCircuitChecker::populate_values(
     values.w_o = builder.get_variable(block.w_o()[idx]);
     // Note: memory_data contains indices into the block to which RAM/ROM gates were added so we need to check that
     // we are indexing into the correct block before updating the w_4 value.
-    if (block.has_ram_rom && memory_data.read_record_gates.contains(idx)) {
+    const bool is_ram_rom_block = (&block == &builder.blocks.aux);
+    if (is_ram_rom_block && memory_data.read_record_gates.contains(idx)) {
         values.w_4 = compute_memory_record_term(
             values.w_l, values.w_r, values.w_o, memory_data.eta, memory_data.eta_two, memory_data.eta_three);
-    } else if (block.has_ram_rom && memory_data.write_record_gates.contains(idx)) {
+    } else if (is_ram_rom_block && memory_data.write_record_gates.contains(idx)) {
         values.w_4 =
             compute_memory_record_term(
                 values.w_l, values.w_r, values.w_o, memory_data.eta, memory_data.eta_two, memory_data.eta_three) +
@@ -296,14 +297,14 @@ void UltraCircuitChecker::populate_values(
         values.w_l_shift = builder.get_variable(block.w_l()[idx + 1]);
         values.w_r_shift = builder.get_variable(block.w_r()[idx + 1]);
         values.w_o_shift = builder.get_variable(block.w_o()[idx + 1]);
-        if (block.has_ram_rom && memory_data.read_record_gates.contains(idx + 1)) {
+        if (is_ram_rom_block && memory_data.read_record_gates.contains(idx + 1)) {
             values.w_4_shift = compute_memory_record_term(values.w_l_shift,
                                                           values.w_r_shift,
                                                           values.w_o_shift,
                                                           memory_data.eta,
                                                           memory_data.eta_two,
                                                           memory_data.eta_three);
-        } else if (block.has_ram_rom && memory_data.write_record_gates.contains(idx + 1)) {
+        } else if (is_ram_rom_block && memory_data.write_record_gates.contains(idx + 1)) {
             values.w_4_shift = compute_memory_record_term(values.w_l_shift,
                                                           values.w_r_shift,
                                                           values.w_o_shift,

barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_format.cpp

@@ -602,40 +602,6 @@ template <> MegaCircuitBuilder create_circuit(AcirProgram& program, const Progra
     return builder;
 };
 
-/**
- * @brief Specialization for creating Ultra circuit from acir constraints and optionally a witness
- *
- * @tparam Builder
- * @param constraint_system
- * @param size_hint
- * @param witness
- * @return Builder
- */
-// TODO(https://github.com/AztecProtocol/barretenberg/issues/1161): Delete this or refactor it.
-template <>
-UltraCircuitBuilder create_circuit(AcirFormat& constraint_system,
-                                   bool recursive,
-                                   const size_t size_hint,
-                                   const WitnessVector& witness,
-                                   uint32_t honk_recursion,
-                                   [[maybe_unused]] std::shared_ptr<ECCOpQueue>,
-                                   bool collect_gates_per_opcode)
-{
-    PROFILE_THIS();
-    Builder builder{ size_hint, witness, constraint_system.public_inputs, constraint_system.varnum, recursive };
-
-    AcirProgram program{ constraint_system, witness };
-    const ProgramMetadata metadata{ .recursive = recursive,
-                                    .honk_recursion = honk_recursion,
-                                    .collect_gates_per_opcode = collect_gates_per_opcode,
-                                    .size_hint = size_hint };
-    build_constraints(builder, program, metadata);
-
-    vinfo("created circuit");
-
-    return builder;
-};
-
 template void build_constraints<MegaCircuitBuilder>(MegaCircuitBuilder&, AcirProgram&, const ProgramMetadata&);
 
 } // namespace acir_format

barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_format.hpp

@@ -223,21 +223,6 @@ struct ProgramMetadata {
 template <typename Builder = bb::UltraCircuitBuilder>
 Builder create_circuit(AcirProgram& program, const ProgramMetadata& metadata = ProgramMetadata{});
 
-// TODO(https://github.com/AztecProtocol/barretenberg/issues/1161) Refactor this function
-template <typename Builder = bb::UltraCircuitBuilder>
-Builder create_circuit(AcirFormat& constraint_system,
-                       // Specifies whether a prover that produces SNARK recursion friendly proofs should be used.
-                       // The proof produced when this flag is true should be friendly for recursive verification inside
-                       // of another SNARK. For example, a recursive friendly proof may use Blake3Pedersen for
-                       // hashing in its transcript, while we still want a prove that uses Keccak for its transcript in
-                       // order to be able to verify SNARKs on Ethereum.
-                       bool recursive,
-                       const size_t size_hint = 0,
-                       const WitnessVector& witness = {},
-                       uint32_t honk_recursion = 0,
-                       std::shared_ptr<bb::ECCOpQueue> op_queue = std::make_shared<bb::ECCOpQueue>(),
-                       bool collect_gates_per_opcode = false);
-
 template <typename Builder>
 void build_constraints(Builder& builder, AcirProgram& program, const ProgramMetadata& metadata);
 

barretenberg/cpp/src/barretenberg/dsl/acir_format/acir_format.test.cpp

@@ -64,7 +64,8 @@ TEST_F(AcirFormatTests, TestASingleConstraintNoPubInputs)
     };
     mock_opcode_indices(constraint_system);
     WitnessVector witness{ 5, 7, 12 };
-    auto builder = create_circuit(constraint_system, /*recursive*/ false, /*size_hint*/ 0, witness);
+    AcirProgram program{ constraint_system, witness };
+    auto builder = create_circuit(program);
 
     EXPECT_TRUE(CircuitChecker::check(builder));
 }
@@ -181,7 +182,8 @@ TEST_F(AcirFormatTests, TestLogicGateFromNoirCircuit)
     WitnessVector witness{
         5, 10, 15, 5, inverse_of_five, 1,
     };
-    auto builder = create_circuit(constraint_system, /*recursive*/ false, /*size_hint*/ 0, witness);
+    AcirProgram program{ constraint_system, witness };
+    auto builder = create_circuit(program);
 
     EXPECT_TRUE(CircuitChecker::check(builder));
 }
@@ -257,7 +259,8 @@ TEST_F(AcirFormatTests, TestKeccakPermutation)
                            18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,
                            35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50 };
 
-    auto builder = create_circuit(constraint_system, /*recursive*/ false, /*size_hint=*/0, witness);
+    AcirProgram program{ constraint_system, witness };
+    auto builder = create_circuit(program);
 
     EXPECT_TRUE(CircuitChecker::check(builder));
 }
@@ -446,7 +449,8 @@ TEST_F(AcirFormatTests, TestBigAdd)
     };
     mock_opcode_indices(constraint_system);
 
-    auto builder = create_circuit(constraint_system, /*recursive*/ false, /*size_hint*/ 0, witness_values);
+    AcirProgram program{ constraint_system, witness_values };
+    auto builder = create_circuit(program);
 
     EXPECT_TRUE(CircuitChecker::check(builder));
 }

barretenberg/cpp/src/barretenberg/dsl/acir_format/bigint_constraint.test.cpp

@@ -203,7 +203,8 @@ TEST_F(BigIntTests, TestBigIntConstraintMultiple)
     mock_opcode_indices(constraint_system);
     constraint_system.varnum = static_cast<uint32_t>(witness.size() + 1);
 
-    auto builder = create_circuit(constraint_system, /*recursive*/ false, /*size_hint*/ 0, witness);
+    AcirProgram program{ constraint_system, witness };
+    auto builder = create_circuit(program);
 
     EXPECT_TRUE(CircuitChecker::check(builder));
 }
@@ -266,7 +267,8 @@ TEST_F(BigIntTests, TestBigIntConstraintSimple)
     WitnessVector witness{
         0, 3, 6, 3, 0,
     };
-    auto builder = create_circuit(constraint_system, /*recursive*/ false, /*size_hint*/ 0, witness);
+    AcirProgram program{ constraint_system, witness };
+    auto builder = create_circuit(program);
 
     EXPECT_TRUE(CircuitChecker::check(builder));
 }
@@ -323,7 +325,8 @@ TEST_F(BigIntTests, TestBigIntConstraintReuse)
     constraint_system.varnum = static_cast<uint32_t>(witness.size() + 1);
     mock_opcode_indices(constraint_system);
 
-    auto builder = create_circuit(constraint_system, /*recursive*/ false, /*size_hint*/ 0, witness);
+    AcirProgram program{ constraint_system, witness };
+    auto builder = create_circuit(program);
 
     EXPECT_TRUE(CircuitChecker::check(builder));
 }
@@ -378,7 +381,8 @@ TEST_F(BigIntTests, TestBigIntConstraintReuse2)
     constraint_system.varnum = static_cast<uint32_t>(witness.size() + 1);
     mock_opcode_indices(constraint_system);
 
-    auto builder = create_circuit(constraint_system, /*recursive*/ false, /*size_hint*/ 0, witness);
+    AcirProgram program{ constraint_system, witness };
+    auto builder = create_circuit(program);
 
     EXPECT_TRUE(CircuitChecker::check(builder));
 }
@@ -448,7 +452,8 @@ TEST_F(BigIntTests, TestBigIntDIV)
     WitnessVector witness{
         0, 6, 3, 2, 0,
     };
-    auto builder = create_circuit(constraint_system, /*recursive*/ false, /*size_hint*/ 0, witness);
+    AcirProgram program{ constraint_system, witness };
+    auto builder = create_circuit(program);
 
     EXPECT_TRUE(CircuitChecker::check(builder));
 }

barretenberg/cpp/src/barretenberg/dsl/acir_format/ec_operations.test.cpp

@@ -89,7 +89,8 @@ TEST_F(EcOperations, TestECOperations)
     };
     mock_opcode_indices(constraint_system);
 
-    auto builder = create_circuit(constraint_system, /*recursive*/ false, /*size_hint*/ 0, witness_values);
+    AcirProgram program{ constraint_system, witness_values };
+    auto builder = create_circuit(program);
 
     EXPECT_TRUE(CircuitChecker::check(builder));
 }
@@ -217,7 +218,8 @@ TEST_F(EcOperations, TestECMultiScalarMul)
     };
     mock_opcode_indices(constraint_system);
 
-    auto builder = create_circuit(constraint_system, /*recursive*/ false, /*size_hint*/ 0, witness_values);
+    AcirProgram program{ constraint_system, witness_values };
+    auto builder = create_circuit(program);
 
     EXPECT_TRUE(CircuitChecker::check(builder));
 }