feat: merge-train/barretenberg (#16850)

BEGIN_COMMIT_OVERRIDE
chore: audit of lookup relations in the ECCVM (#16712)
feat: translator handles no-op range (#16628)
chore: audit wnaf relations in the ECCVM (#16573)
feat: add option to choose between tmpfs and $HOME/tmp in docker_isolate
(#16854)
refactor(bb-prover): don't use --write_vk in yarn-project (#16834)
chore: cycle group cleanup (#16830)
END_COMMIT_OVERRIDE

Author: AztecBot

barretenberg/cpp/scripts/test_civc_standalone_vks_havent_changed.sh

@@ -13,8 +13,7 @@ cd ..
 # - Generate a hash for versioning: sha256sum bb-civc-inputs.tar.gz
 # - Upload the compressed results: aws s3 cp bb-civc-inputs.tar.gz s3://aztec-ci-artifacts/protocol/bb-civc-inputs-[hash(0:8)].tar.gz
 # Note: In case of the "Test suite failed to run ... Unexpected token 'with' " error, need to run: docker pull aztecprotocol/build:3.0
-
-pinned_short_hash="ec9b5be3"
+pinned_short_hash="16b530e4"
 pinned_civc_inputs_url="https://aztec-ci-artifacts.s3.us-east-2.amazonaws.com/protocol/bb-civc-inputs-${pinned_short_hash}.tar.gz"
 
 function compress_and_upload {

barretenberg/cpp/src/barretenberg/boomerang_value_detection/graph_description_goblin.test.cpp

@@ -40,37 +40,25 @@ class BoomerangGoblinRecursiveVerifierTests : public testing::Test {
      *
      * @return ProverOutput
      */
-    static ProverOutput create_goblin_prover_output(const size_t NUM_CIRCUITS = 3)
+    static ProverOutput create_goblin_prover_output()
     {
         Goblin goblin;
-        // Construct and accumulate multiple circuits
-        for (size_t idx = 0; idx < NUM_CIRCUITS - 1; ++idx) {
-            MegaCircuitBuilder builder{ goblin.op_queue };
-            GoblinMockCircuits::construct_simple_circuit(builder);
-            goblin.prove_merge();
-        }
-
-        auto goblin_transcript = std::make_shared<Goblin::Transcript>();
+        GoblinMockCircuits::construct_and_merge_mock_circuits(goblin, 5);
 
-        Goblin goblin_final;
-        goblin_final.op_queue = goblin.op_queue;
-        MegaCircuitBuilder builder{ goblin_final.op_queue };
-        GoblinMockCircuits::construct_simple_circuit(builder, /*last_circuit=*/true);
-        goblin_final.op_queue->merge();
+        // Merge the ecc ops from the newly constructed circuit
+        auto goblin_proof = goblin.prove(MergeSettings::APPEND);
         // Subtable values and commitments - needed for (Recursive)MergeVerifier
         MergeCommitments merge_commitments;
-        auto t_current = goblin_final.op_queue->construct_current_ultra_ops_subtable_columns();
-        auto T_prev = goblin_final.op_queue->construct_previous_ultra_ops_table_columns();
-        CommitmentKey<curve::BN254> pcs_commitment_key(goblin_final.op_queue->get_ultra_ops_table_num_rows());
+        auto t_current = goblin.op_queue->construct_current_ultra_ops_subtable_columns();
+        auto T_prev = goblin.op_queue->construct_previous_ultra_ops_table_columns();
+        CommitmentKey<curve::BN254> pcs_commitment_key(goblin.op_queue->get_ultra_ops_table_num_rows());
         for (size_t idx = 0; idx < MegaFlavor::NUM_WIRES; idx++) {
             merge_commitments.t_commitments[idx] = pcs_commitment_key.commit(t_current[idx]);
             merge_commitments.T_prev_commitments[idx] = pcs_commitment_key.commit(T_prev[idx]);
         }
 
         // Output is a goblin proof plus ECCVM/Translator verification keys
-        return { goblin_final.prove(),
-                 { std::make_shared<ECCVMVK>(), std::make_shared<TranslatorVK>() },
-                 merge_commitments };
+        return { goblin_proof, { std::make_shared<ECCVMVK>(), std::make_shared<TranslatorVK>() }, merge_commitments };
     }
 };
 
@@ -96,7 +84,7 @@ TEST_F(BoomerangGoblinRecursiveVerifierTests, graph_description_basic)
     }
 
     GoblinRecursiveVerifier verifier{ &builder, verifier_input };
-    GoblinRecursiveVerifierOutput output = verifier.verify(proof, recursive_merge_commitments);
+    GoblinRecursiveVerifierOutput output = verifier.verify(proof, recursive_merge_commitments, MergeSettings::APPEND);
     output.points_accumulator.set_public();
     // Construct and verify a proof for the Goblin Recursive Verifier circuit
     {

barretenberg/cpp/src/barretenberg/circuit_checker/translator_circuit_checker.cpp

@@ -268,6 +268,9 @@ ClientIVC::perform_recursive_verification_and_databus_consistency_checks(
     pairing_points.aggregate(nested_pairing_points);
     if (is_hiding_kernel) {
         pairing_points.aggregate(decider_pairing_points);
+        // Placeholder for randomness at the end of the hiding circuit (to be handled in subsequent PR)
+        circuit.queue_ecc_no_op();
+        circuit.queue_ecc_no_op();
     }
 
     return { output_verifier_accumulator, pairing_points, merged_table_commitments };
@@ -310,6 +313,12 @@ void ClientIVC::complete_kernel_circuit_logic(ClientCircuit& circuit)
     // to ensure the op queue wires in translator are shiftable, i.e. their 0th coefficient is 0. (The tail kernel
     // subtable is at the top of the final aggregate table since it is the last to be prepended).
     if (is_tail_kernel) {
+        BB_ASSERT_EQ(circuit.op_queue->get_current_subtable_size(),
+                     0U,
+                     "tail kernel ecc ops table should be empty at this point");
+        circuit.queue_ecc_no_op();
+        // Placeholder for randomness at the beginning of tail circuit
+        circuit.queue_ecc_no_op();
         circuit.queue_ecc_no_op();
     }
     circuit.queue_ecc_eq();

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp

@@ -8,6 +8,13 @@ namespace bb {
 // permutation argument polynomials (sigmas, ids) are unique, e.g. id[i][j] == id[m][n] iff (i == m && j == n)
 constexpr uint32_t PERMUTATION_ARGUMENT_VALUE_SEPARATOR = 1 << 28;
 
+// The fixed size of the Translator trace where each accumulation gate, corresponding to one UltraOp, will occupy two
+// rows.
+static constexpr uint32_t CONST_TRANSLATOR_MINI_CIRCUIT_LOG_SIZE = 14;
+
+// -1 as each op occupies two rows in Translator trace
+static constexpr uint32_t CONST_OP_QUEUE_LOG_SIZE = CONST_TRANSLATOR_MINI_CIRCUIT_LOG_SIZE - 1;
+
 // The log of the max circuit size assumed in order to achieve constant sized Honk proofs
 // TODO(https://github.com/AztecProtocol/barretenberg/issues/1046): Remove the need for const sized proofs
 static constexpr uint32_t CONST_PROOF_SIZE_LOG_N = 28;

barretenberg/cpp/src/barretenberg/constants.hpp

@@ -512,7 +512,7 @@ class ECCVMFlavor {
          *          transcript_msm_count_at_transition_inverse: used to validate transcript_msm_count_zero_at_transition
          *          precompute_pc: point counter for Straus precomputation columns
          *          precompute_select: if 1, evaluate Straus precomputation algorithm at current row
-         *          precompute_point_transition: 1 if current row operating on a different point to previous row
+         *          precompute_point_transition: 1 if next row operating on a different point than current row.
          *          precompute_round: round counter for Straus precomputation algorithm
          *          precompute_scalar_sum: accumulating sum of Straus scalar slices
          *          precompute_s1hi/lo: 2-bit hi/lo components of a Straus 4-bit scalar slice

barretenberg/cpp/src/barretenberg/eccvm/eccvm_flavor.hpp

@@ -12,6 +12,7 @@
 #include "barretenberg/crypto/merkle_tree/memory_store.hpp"
 #include "barretenberg/crypto/merkle_tree/merkle_tree.hpp"
 #include "barretenberg/flavor/mega_flavor.hpp"
+#include "barretenberg/goblin/goblin.hpp"
 #include "barretenberg/srs/global_crs.hpp"
 #include "barretenberg/stdlib/encryption/ecdsa/ecdsa.hpp"
 #include "barretenberg/stdlib/hash/keccak/keccak.hpp"
@@ -136,19 +137,34 @@ class GoblinMockCircuits {
      *
      * @param builder
      */
-    static void construct_simple_circuit(MegaBuilder& builder, bool last_circuit = false)
+    static void construct_simple_circuit(MegaBuilder& builder)
     {
         BB_BENCH();
-        // The last circuit to be accumulated must contain a no-op
-        if (last_circuit) {
-            builder.queue_ecc_no_op();
-        }
 
         add_some_ecc_op_gates(builder);
         MockCircuits::construct_arithmetic_circuit(builder);
         bb::stdlib::recursion::honk::DefaultIO<MegaBuilder>::add_default(builder);
     }
 
+    static void construct_and_merge_mock_circuits(Goblin& goblin, const size_t num_circuits = 3)
+    {
+        for (size_t idx = 0; idx < num_circuits - 1; ++idx) {
+            MegaCircuitBuilder builder{ goblin.op_queue };
+            if (idx == num_circuits - 2) {
+                // Last circuit appended needs to begin with a no-op for translator to be shiftable
+                builder.queue_ecc_no_op();
+            }
+            construct_simple_circuit(builder);
+            goblin.prove_merge();
+            // Pop the merge proof from the queue, Goblin will be verified at the end
+            goblin.merge_verification_queue.pop_front();
+        }
+        MegaCircuitBuilder builder{ goblin.op_queue };
+        GoblinMockCircuits::construct_simple_circuit(builder);
+        builder.queue_ecc_no_op();
+        builder.queue_ecc_no_op();
+    }
+
     /**
      * @brief Construct a mock kernel circuit
      * @details Construct an arbitrary circuit meant to represent the aztec private function execution kernel. Recursive

barretenberg/cpp/src/barretenberg/goblin/mock_circuits.hpp

@@ -48,6 +48,7 @@ class ECCOpQueue {
     EccvmRowTracker eccvm_row_tracker;
 
   public:
+    static const size_t OP_QUEUE_SIZE = 1 << CONST_OP_QUEUE_LOG_SIZE;
     /**
      * @brief Instantiate an initial ECC op subtable.
      */
@@ -63,6 +64,8 @@ class ECCOpQueue {
         ultra_ops_table.create_new_subtable();
     }
 
+    size_t get_current_subtable_size() const { return ultra_ops_table.get_current_subtable_size(); }
+
     void merge(MergeSettings settings = MergeSettings::PREPEND, std::optional<size_t> ultra_fixed_offset = std::nullopt)
     {
         eccvm_ops_table.merge(settings);
@@ -75,7 +78,8 @@ class ECCOpQueue {
         return ultra_ops_table.construct_table_columns();
     }
 
-    // Construct polys corresponding to the columns of the aggregate ultra ops table, excluding the most recent subtable
+    // Construct polys corresponding to the columns of the aggregate ultra ops table, excluding the most recent
+    // subtable
     std::array<Polynomial<Fr>, ULTRA_TABLE_WIDTH> construct_previous_ultra_ops_table_columns() const
     {
         return ultra_ops_table.construct_previous_table_columns();
@@ -97,8 +101,8 @@ class ECCOpQueue {
     size_t get_current_ultra_ops_subtable_num_rows() const { return ultra_ops_table.current_ultra_subtable_size(); }
     size_t get_previous_ultra_ops_table_num_rows() const { return ultra_ops_table.previous_ultra_table_size(); }
 
-    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1339): Consider making the ultra and eccvm ops getters
-    // more memory efficient
+    // TODO(https://github.com/AztecProtocol/barretenberg/issues/1339): Consider making the ultra and eccvm ops
+    // getters more memory efficient
 
     // Get the full table of ECCVM ops in contiguous memory; construct it if it has not been constructed already
     std::vector<ECCVMOperation>& get_eccvm_ops()
@@ -225,8 +229,8 @@ class ECCOpQueue {
     /**
      * @brief Writes randomness to the ultra ops table but adds no eccvm operations.
      *
-     * @details This method is used to add randomness to the ultra ops table with the aim of randomising the commitment
-     * and evaluations of its corresponding columns
+     * @details This method is used to add randomness to the ultra ops table with the aim of randomising the
+     * commitment and evaluations of its corresponding columns
      * @return UltraOp
      */
     UltraOp random_op_ultra_only()

barretenberg/cpp/src/barretenberg/op_queue/ecc_op_queue.hpp

@@ -54,7 +54,9 @@ class ECCOpQueueTest {
             } else {
                 // APPEND merge performs concatenation directly to end of previous table or at a specified fixed offset
                 const size_t prev_table_size = op_queue->get_previous_ultra_ops_table_num_rows(); // k
-                const size_t shift_magnitude = ultra_fixed_offset.value_or(prev_table_size);
+                const size_t shift_magnitude = ultra_fixed_offset.has_value()
+                                                   ? ultra_fixed_offset.value() * bb::UltraEccOpsTable::NUM_ROWS_PER_OP
+                                                   : prev_table_size; // k
                 // T(x) = T_prev(x) + x^k * t_current(x), where k is the shift magnitude
                 const Fr prev_table_eval = prev_table_poly.evaluate(eval_challenge); // T_prev(x)
                 const Fr shifted_subtable_eval =
@@ -74,10 +76,12 @@ class ECCOpQueueTest {
         auto ultra_table = op_queue->get_ultra_ops();
         auto eccvm_table = op_queue->get_eccvm_ops();
 
-        EXPECT_EQ(eccvm_table.size(), ultra_table.size());
-
-        for (auto [ultra_op, eccvm_op] : zip_view(ultra_table, eccvm_table)) {
-            EXPECT_EQ(ultra_op.op_code.value(), eccvm_op.op_code.value());
+        size_t j = 0;
+        for (const auto& ultra_op : ultra_table) {
+            if (ultra_op.op_code.value() == 0) {
+                continue;
+            }
+            EXPECT_EQ(ultra_op.op_code.value(), eccvm_table[j++].op_code.value());
         }
     };
 };

barretenberg/cpp/src/barretenberg/op_queue/ecc_op_queue.test.cpp

@@ -74,6 +74,8 @@ struct UltraOp {
     Fr z_2;
     bool return_is_infinity;
 
+    bool operator==(const UltraOp& other) const = default;
+
     /**
      * @brief Get the point in standard form i.e. as two coordinates x and y in the base field or as a point at
      * infinity whose coordinates are set to (0,0).
@@ -130,6 +132,7 @@ template <typename OpFormat> class EccOpsTable {
     }
 
     size_t num_subtables() const { return table.size(); }
+    size_t get_current_subtable_size() const { return current_subtable.size(); }
 
     auto& get() const { return table; }
 
@@ -246,7 +249,7 @@ class UltraEccOpsTable {
                 // The last subtable in deque is the fixed-location one
                 last_subtable_size = table.get().back().size() * NUM_ROWS_PER_OP;
             }
-            return std::max(base_size, fixed_append_offset.value() + last_subtable_size);
+            return std::max(base_size, (fixed_append_offset.value() * NUM_ROWS_PER_OP) + last_subtable_size);
         }
         return base_size;
     }
@@ -270,7 +273,46 @@ class UltraEccOpsTable {
         }
     }
 
-    std::vector<UltraOp> get_reconstructed() const { return table.get_reconstructed(); }
+    size_t get_current_subtable_size() const { return table.get_current_subtable_size(); }
+
+    std::vector<UltraOp> get_reconstructed() const
+    {
+        if (has_fixed_append && fixed_append_offset.has_value()) {
+            return get_reconstructed_with_fixed_append();
+        }
+        return table.get_reconstructed();
+    }
+    std::vector<UltraOp> get_reconstructed_with_fixed_append() const
+    {
+
+        ASSERT(get_current_subtable_size() == 0,
+               "current subtable should be merged before reconstructing the full table of operations.");
+
+        std::vector<UltraOp> reconstructed_table;
+        reconstructed_table.reserve(1 << CONST_OP_QUEUE_LOG_SIZE);
+
+        for (size_t subtable_idx = 0; subtable_idx < table.num_subtables() - 1; subtable_idx++) {
+            const auto& subtable = table.get()[subtable_idx];
+            for (const auto& op : subtable) {
+                reconstructed_table.push_back(op);
+            }
+        }
+
+        // Add zeros if fixed offset is larger than current size
+        if (has_fixed_append && fixed_append_offset.has_value()) {
+            size_t current_size = reconstructed_table.size();
+            size_t target_offset = fixed_append_offset.value();
+            // Fill gap with no-ops if needed
+            reconstructed_table.insert(reconstructed_table.end(), target_offset - current_size, UltraOp{ /*no-op*/ });
+        }
+
+        // Add the final subtable (appended at fixed location)
+        const auto& final_subtable = table.get()[table.num_subtables() - 1];
+        for (const auto& op : final_subtable) {
+            reconstructed_table.push_back(op);
+        }
+        return reconstructed_table;
+    }
 
     // Construct the columns of the full ultra ecc ops table
     ColumnPolynomials construct_table_columns() const
@@ -342,7 +384,7 @@ class UltraEccOpsTable {
 
         // Process all prepended subtables (all except last)
         size_t i = 0;
-        for (size_t subtable_idx = 0; subtable_idx < table.num_subtables() - 1; ++subtable_idx) {
+        for (size_t subtable_idx = 0; subtable_idx < table.num_subtables() - 1; subtable_idx++) {
             const auto& subtable = table.get()[subtable_idx];
             for (const auto& op : subtable) {
                 write_op_to_polynomials(column_polynomials, op, i);
@@ -351,7 +393,7 @@ class UltraEccOpsTable {
         }
 
         // Place the appended subtable at the fixed offset
-        size_t append_position = fixed_append_offset.value_or(i);
+        size_t append_position = fixed_append_offset.has_value() ? fixed_append_offset.value() * NUM_ROWS_PER_OP : i;
         const auto& appended_subtable = table.get()[table.num_subtables() - 1];
 
         size_t j = append_position;