feat: merge-train/barretenberg (#16334)

See
[merge-train-readme.md](https://github.com/AztecProtocol/aztec-packages/blob/next/.github/workflows/merge-train-readme.md).

BEGIN_COMMIT_OVERRIDE
feat!: fiat-shamir PG accumulator (#16243)
fix vk check test
END_COMMIT_OVERRIDE

---------

Co-authored-by: AztecBot <tech@aztecprotocol.com>
Co-authored-by: Lucas Xia <lucasxia01@gmail.com>

Author: 3 people

barretenberg/cpp/scripts/test_civc_standalone_vks_havent_changed.sh

@@ -11,7 +11,7 @@ cd ..
 # - Generate a hash for versioning: sha256sum bb-civc-inputs.tar.gz
 # - Upload the compressed results: aws s3 cp bb-civc-inputs.tar.gz s3://aztec-ci-artifacts/protocol/bb-civc-inputs-[hash(0:8)].tar.gz
 # Note: In case of the "Test suite failed to run ... Unexpected token 'with' " error, need to run: docker pull aztecprotocol/build:3.0
-pinned_short_hash="9e41a005"
+pinned_short_hash="42c72e0d"
 pinned_civc_inputs_url="https://aztec-ci-artifacts.s3.us-east-2.amazonaws.com/protocol/bb-civc-inputs-${pinned_short_hash}.tar.gz"
 
 function compress_and_upload {

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.cpp

@@ -11,6 +11,7 @@
 #include "barretenberg/serialize/msgpack_impl.hpp"
 #include "barretenberg/special_public_inputs/special_public_inputs.hpp"
 #include "barretenberg/ultra_honk/oink_prover.hpp"
+#include "barretenberg/ultra_honk/oink_verifier.hpp"
 
 namespace bb {
 
@@ -112,7 +113,8 @@ std::pair<ClientIVC::PairingPoints, ClientIVC::TableCommitments> ClientIVC::
     case QUEUE_TYPE::PG_TAIL:
     case QUEUE_TYPE::PG: {
         // Construct stdlib verifier accumulator from the native counterpart computed on a previous round
-        auto stdlib_verifier_accum = std::make_shared<RecursiveDeciderVerificationKey>(&circuit, verifier_accumulator);
+        auto stdlib_verifier_accum =
+            std::make_shared<RecursiveDeciderVerificationKey>(&circuit, recursive_verifier_native_accum);
 
         // Perform folding recursive verification to update the verifier accumulator
         FoldingRecursiveVerifier verifier{
@@ -121,7 +123,7 @@ std::pair<ClientIVC::PairingPoints, ClientIVC::TableCommitments> ClientIVC::
         auto verifier_accum = verifier.verify_folding_proof(verifier_inputs.proof);
 
         // Extract native verifier accumulator from the stdlib accum for use on the next round
-        verifier_accumulator = std::make_shared<DeciderVerificationKey>(verifier_accum->get_value());
+        recursive_verifier_native_accum = std::make_shared<DeciderVerificationKey>(verifier_accum->get_value());
 
         witness_commitments = std::move(verifier.keys_to_fold[1]->witness_commitments);
         public_inputs = std::move(verifier.public_inputs);
@@ -140,9 +142,9 @@ std::pair<ClientIVC::PairingPoints, ClientIVC::TableCommitments> ClientIVC::
         verifier_accum->is_accumulator = true; // indicate to PG that it should not run oink
 
         // Extract native verifier accumulator from the stdlib accum for use on the next round
-        verifier_accumulator = std::make_shared<DeciderVerificationKey>(verifier_accum->get_value());
+        recursive_verifier_native_accum = std::make_shared<DeciderVerificationKey>(verifier_accum->get_value());
         // Initialize the gate challenges to zero for use in first round of folding
-        verifier_accumulator->gate_challenges = std::vector<FF>(CONST_PG_LOG_N, 0);
+        recursive_verifier_native_accum->gate_challenges = std::vector<FF>(CONST_PG_LOG_N, 0);
 
         witness_commitments = std::move(verifier_accum->witness_commitments);
         public_inputs = std::move(verifier.public_inputs);
@@ -321,7 +323,7 @@ void ClientIVC::accumulate(ClientCircuit& circuit, const std::shared_ptr<MegaVer
     // Transcript to be shared across folding of K_{i} (kernel) (the current kernel), A_{i+1,1} (app), .., A_{i+1,
     // n} (app)
     if (is_kernel) {
-        accumulation_transcript = std::make_shared<Transcript>();
+        prover_accumulation_transcript = std::make_shared<Transcript>();
     }
 
     VerifierInputs queue_entry{ .honk_vk = honk_vk,
@@ -330,7 +332,9 @@ void ClientIVC::accumulate(ClientCircuit& circuit, const std::shared_ptr<MegaVer
     if (num_circuits_accumulated == 0) { // First circuit in the IVC
         BB_ASSERT_EQ(queue_entry.is_kernel, false, "First circuit accumulated is always be an app");
         // For first circuit in the IVC, use oink to complete the decider proving key and generate an oink proof
-        MegaOinkProver oink_prover{ proving_key, honk_vk, accumulation_transcript };
+        auto oink_verifier_transcript =
+            Transcript::convert_prover_transcript_to_verifier_transcript(prover_accumulation_transcript);
+        MegaOinkProver oink_prover{ proving_key, honk_vk, prover_accumulation_transcript };
         vinfo("computing oink proof...");
         oink_prover.prove();
         HonkProof oink_proof = oink_prover.export_proof();
@@ -341,17 +345,31 @@ void ClientIVC::accumulate(ClientCircuit& circuit, const std::shared_ptr<MegaVer
 
         fold_output.accumulator = proving_key; // initialize the prover accum with the completed key
 
+        auto decider_vk = std::make_shared<DeciderVerificationKey>(honk_vk);
+        oink_verifier_transcript->load_proof(oink_proof);
+        OinkVerifier<Flavor> oink_verifier{ decider_vk, oink_verifier_transcript };
+        oink_verifier.verify();
+        native_verifier_accum = decider_vk;
+        native_verifier_accum->is_accumulator = true;
+        native_verifier_accum->gate_challenges = std::vector<FF>(CONST_PG_LOG_N, 0);
+
         queue_entry.type = QUEUE_TYPE::OINK;
         queue_entry.proof = oink_proof;
     } else { // Otherwise, fold the new key into the accumulator
         vinfo("computing folding proof");
         auto vk = std::make_shared<DeciderVerificationKey_<Flavor>>(honk_vk);
+        // make a copy of the prover_accumulation_transcript for the verifier to use
+        auto verifier_accumulation_transcript =
+            Transcript::convert_prover_transcript_to_verifier_transcript(prover_accumulation_transcript);
+
         FoldingProver folding_prover({ fold_output.accumulator, proving_key },
-                                     { verifier_accumulator, vk },
-                                     accumulation_transcript,
+                                     { native_verifier_accum, vk },
+                                     prover_accumulation_transcript,
                                      trace_usage_tracker);
         fold_output = folding_prover.prove();
         vinfo("constructed folding proof");
+        FoldingVerifier folding_verifier({ native_verifier_accum, vk }, verifier_accumulation_transcript);
+        native_verifier_accum = folding_verifier.verify_folding_proof(fold_output.proof);
 
         if (num_circuits_accumulated == num_circuits - 1) {
             // we are folding in the "Tail" kernel, so the verification_queue entry should have type PG_FINAL
@@ -369,7 +387,7 @@ void ClientIVC::accumulate(ClientCircuit& circuit, const std::shared_ptr<MegaVer
     verification_queue.push_back(queue_entry);
 
     // Construct merge proof for the present circuit
-    goblin.prove_merge(accumulation_transcript);
+    goblin.prove_merge(prover_accumulation_transcript);
 
     num_circuits_accumulated++;
 }
@@ -414,7 +432,7 @@ std::pair<ClientIVC::PairingPoints, ClientIVC::TableCommitments> ClientIVC::comp
 
     // Construct stdlib accumulator, decider vkey and folding proof
     auto stdlib_verifier_accumulator =
-        std::make_shared<RecursiveDeciderVerificationKey>(&circuit, verifier_accumulator);
+        std::make_shared<RecursiveDeciderVerificationKey>(&circuit, recursive_verifier_native_accum);
 
     // Propagate the public inputs of the tail kernel by converting them to public inputs of the hiding circuit.
     auto num_public_inputs = static_cast<size_t>(honk_vk->num_public_inputs);
@@ -427,7 +445,7 @@ std::pair<ClientIVC::PairingPoints, ClientIVC::TableCommitments> ClientIVC::comp
     FoldingRecursiveVerifier folding_verifier{
         &circuit, stdlib_verifier_accumulator, { stdlib_vk_and_hash }, pg_merge_transcript
     };
-    auto recursive_verifier_accumulator = folding_verifier.verify_folding_proof(stdlib_proof);
+    auto recursive_verifier_native_accum = folding_verifier.verify_folding_proof(stdlib_proof);
     verification_queue.clear();
 
     // Get the completed decider verification key corresponding to the tail kernel from the folding verifier
@@ -452,7 +470,7 @@ std::pair<ClientIVC::PairingPoints, ClientIVC::TableCommitments> ClientIVC::comp
     points_accumulator.aggregate(kernel_input.pairing_inputs);
 
     // Perform recursive decider verification
-    DeciderRecursiveVerifier decider{ &circuit, recursive_verifier_accumulator };
+    DeciderRecursiveVerifier decider{ &circuit, recursive_verifier_native_accum };
     BB_ASSERT_EQ(!decider_proof.empty(), true, "Decider proof is empty!");
     PairingPoints decider_pairing_points = decider.verify_proof(decider_proof);
     points_accumulator.aggregate(decider_pairing_points);

barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.hpp

@@ -10,6 +10,7 @@
 #include "barretenberg/goblin/goblin.hpp"
 #include "barretenberg/honk/execution_trace/execution_trace_usage_tracker.hpp"
 #include "barretenberg/protogalaxy/protogalaxy_prover.hpp"
+#include "barretenberg/protogalaxy/protogalaxy_verifier.hpp"
 #include "barretenberg/stdlib/honk_verifier/decider_recursive_verifier.hpp"
 #include "barretenberg/stdlib/honk_verifier/oink_recursive_verifier.hpp"
 #include "barretenberg/stdlib/honk_verifier/ultra_recursive_verifier.hpp"
@@ -53,6 +54,7 @@ class ClientIVC {
     using DeciderProvingKeys = DeciderProvingKeys_<Flavor>;
     using FoldingProver = ProtogalaxyProver_<Flavor>;
     using DeciderVerificationKeys = DeciderVerificationKeys_<Flavor>;
+    using FoldingVerifier = ProtogalaxyVerifier_<DeciderVerificationKeys>;
     using ECCVMVerificationKey = bb::ECCVMFlavor::VerificationKey;
     using TranslatorVerificationKey = bb::TranslatorFlavor::VerificationKey;
     using MegaProver = UltraProver_<Flavor>;
@@ -166,7 +168,7 @@ class ClientIVC {
     std::shared_ptr<Transcript> transcript = std::make_shared<Transcript>();
 
     // Transcript to be shared across the folding of K_{i-1} (kernel), A_{i,1} (app), .., A_{i, n}
-    std::shared_ptr<Transcript> accumulation_transcript = std::make_shared<Transcript>();
+    std::shared_ptr<Transcript> prover_accumulation_transcript = std::make_shared<Transcript>();
 
     std::unique_ptr<ClientCircuit> hiding_circuit;
 
@@ -178,7 +180,10 @@ class ClientIVC {
     HonkProof decider_proof;      // decider proof to be verified in the hiding circuit
     HonkProof mega_proof;         // proof of the hiding circuit
 
-    std::shared_ptr<DeciderVerificationKey> verifier_accumulator; // verifier accumulator
+    std::shared_ptr<DeciderVerificationKey>
+        recursive_verifier_native_accum; // native verifier accumulator used in recursive folding
+    std::shared_ptr<DeciderVerificationKey>
+        native_verifier_accum;                    //  native verifier accumulator used in prover folding
     std::shared_ptr<MegaVerificationKey> honk_vk; // honk vk to be completed and folded into the accumulator
 
     // Set of tuples {proof, verification_key, type (Oink/PG)} to be recursively verified

barretenberg/cpp/src/barretenberg/dsl/acir_format/ivc_recursion_constraint.cpp

@@ -57,14 +57,14 @@ std::shared_ptr<ClientIVC> create_mock_ivc_from_constraints(const std::vector<Re
 
     // Case: RESET kernel; single PG recursive verification of a kernel
     if (constraints.size() == 1 && constraints[0].proof_type == pg_type) {
-        ivc->verifier_accumulator = create_mock_decider_vk<ClientIVC::Flavor>();
+        ivc->recursive_verifier_native_accum = create_mock_decider_vk<ClientIVC::Flavor>();
         mock_ivc_accumulation(ivc, ClientIVC::QUEUE_TYPE::PG, /*is_kernel=*/true);
         return ivc;
     }
 
     // Case: TAIL kernel; single PG recursive verification of a kernel
     if (constraints.size() == 1 && constraints[0].proof_type == pg_tail_type) {
-        ivc->verifier_accumulator = create_mock_decider_vk<ClientIVC::Flavor>();
+        ivc->recursive_verifier_native_accum = create_mock_decider_vk<ClientIVC::Flavor>();
         mock_ivc_accumulation(ivc, ClientIVC::QUEUE_TYPE::PG_TAIL, /*is_kernel=*/true);
         return ivc;
     }
@@ -73,20 +73,20 @@ std::shared_ptr<ClientIVC> create_mock_ivc_from_constraints(const std::vector<Re
     if (constraints.size() == 2) {
         BB_ASSERT_EQ(constraints[0].proof_type, pg_type);
         BB_ASSERT_EQ(constraints[1].proof_type, pg_type);
-        ivc->verifier_accumulator = create_mock_decider_vk<ClientIVC::Flavor>();
+        ivc->recursive_verifier_native_accum = create_mock_decider_vk<ClientIVC::Flavor>();
         mock_ivc_accumulation(ivc, ClientIVC::QUEUE_TYPE::PG, /*is_kernel=*/true);
         mock_ivc_accumulation(ivc, ClientIVC::QUEUE_TYPE::PG, /*is_kernel=*/false);
         return ivc;
     }
 
     // Case: HIDING kernel; single PG_FINAL recursive verification of a kernel
     if (constraints.size() == 1 && constraints[0].proof_type == pg_final_type) {
-        ivc->verifier_accumulator = create_mock_decider_vk<ClientIVC::Flavor>();
+        ivc->recursive_verifier_native_accum = create_mock_decider_vk<ClientIVC::Flavor>();
 
         // TODO(https://github.com/AztecProtocol/barretenberg/issues/1283): We need to set the log circuit size here due
         // to an invalid out of circuit max operation in the PG recursive verifier. Once that is resolved this should
         // not be necessary.
-        ivc->verifier_accumulator->vk->log_circuit_size = 18;
+        ivc->recursive_verifier_native_accum->vk->log_circuit_size = 18;
         mock_ivc_accumulation(ivc, ClientIVC::QUEUE_TYPE::PG_FINAL, /*is_kernel=*/true);
         return ivc;
     }

barretenberg/cpp/src/barretenberg/flavor/flavor.hpp

@@ -226,7 +226,9 @@ class NativeVerificationKey_ : public PrecomputedCommitments {
             transcript.add_to_independent_hash_buffer(domain_separator + "vk_commitment", commitment);
         }
 
-        return transcript.hash_independent_buffer(domain_separator + "vk_hash");
+        fr vk_hash = transcript.hash_independent_buffer();
+        transcript.add_to_hash_buffer(domain_separator + "vk_hash", vk_hash);
+        return vk_hash;
     };
 };
 
@@ -316,8 +318,9 @@ class StdlibVerificationKey_ : public PrecomputedCommitments {
         for (const Commitment& commitment : this->get_all()) {
             transcript.add_to_independent_hash_buffer(domain_separator + "vk_commitment", commitment);
         }
-
-        return transcript.hash_independent_buffer(domain_separator + "vk_hash");
+        FF vk_hash = transcript.hash_independent_buffer();
+        transcript.add_to_hash_buffer(domain_separator + "vk_hash", vk_hash);
+        return vk_hash;
     };
 };
 

barretenberg/cpp/src/barretenberg/flavor/native_verification_key.test.cpp

@@ -79,7 +79,7 @@ TYPED_TEST(NativeVerificationKeyTests, VKHashingConsistency)
     for (const auto& field_element : vk_field_elements) {
         transcript.add_to_independent_hash_buffer("vk_element", field_element);
     }
-    fr vkey_hash_1 = transcript.hash_independent_buffer("vk_hash");
+    fr vkey_hash_1 = transcript.hash_independent_buffer();
     // Second method of hashing: using hash().
     fr vkey_hash_2 = vk.hash();
     EXPECT_EQ(vkey_hash_1, vkey_hash_2);

barretenberg/cpp/src/barretenberg/flavor/stdlib_verification_key.test.cpp

@@ -77,7 +77,7 @@ TYPED_TEST(StdlibVerificationKeyTests, VKHashingConsistency)
     for (const auto& field_element : vk_field_elements) {
         transcript.add_to_independent_hash_buffer("vk_element", field_element);
     }
-    FF vkey_hash_1 = transcript.hash_independent_buffer("vk_hash");
+    FF vkey_hash_1 = transcript.hash_independent_buffer();
     // Second method of hashing: using hash().
     FF vkey_hash_2 = vk.hash(outer_builder);
     EXPECT_EQ(vkey_hash_1.get_value(), vkey_hash_2.get_value());

barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy_prover_impl.hpp

@@ -32,11 +32,15 @@ void ProtogalaxyProver_<Flavor, NUM_KEYS>::run_oink_prover_on_each_incomplete_ke
     size_t idx = 0;
     auto& key = keys_to_fold[0];
     auto domain_separator = std::to_string(idx);
-    auto& vk = vks_to_fold[0];
+    auto& verifier_accum = vks_to_fold[0];
     if (!key->is_accumulator) {
-        run_oink_prover_on_one_incomplete_key(key, vk, domain_separator);
+        run_oink_prover_on_one_incomplete_key(key, verifier_accum, domain_separator);
         key->target_sum = 0;
         key->gate_challenges = std::vector<FF>(CONST_PG_LOG_N, 0);
+    } else {
+        // Fiat-Shamir the verifier accumulator
+        FF accum_hash = verifier_accum->add_hash_to_transcript("", *transcript);
+        info("Accumulator hash in PG prover: ", accum_hash);
     }
 
     idx++;

barretenberg/cpp/src/barretenberg/protogalaxy/protogalaxy_verifier.cpp

@@ -24,6 +24,10 @@ void ProtogalaxyVerifier_<DeciderVerificationKeys>::run_oink_verifier_on_each_in
         oink_verifier.verify();
         key->target_sum = 0;
         key->gate_challenges = std::vector<FF>(CONST_PG_LOG_N, 0);
+    } else {
+        // Fiat-Shamir the verifier accumulator
+        FF accum_hash = key->add_hash_to_transcript("", *transcript);
+        info("Accumulator hash in PG verifier: ", accum_hash);
     }
 
     key = keys_to_fold[1];

barretenberg/cpp/src/barretenberg/stdlib/protogalaxy_verifier/protogalaxy_recursive_verifier.cpp

@@ -26,6 +26,12 @@ void ProtogalaxyRecursiveVerifier_<DeciderVerificationKeys>::run_oink_verifier_o
         oink_verifier.verify();
         key->target_sum = 0;
         key->gate_challenges = std::vector<FF>(CONST_PG_LOG_N, 0);
+    } else {
+        // Fiat-Shamir the accumulator.
+        // TODO(https://github.com/AztecProtocol/barretenberg/issues/1390): assert_equal on accumulator hash with public
+        // input hash.
+        FF accum_hash = key->add_hash_to_transcript("", *transcript);
+        info("Accumulator hash in PG rec verifier: ", accum_hash);
     }
 
     key = keys_to_fold[1];
@@ -61,8 +67,8 @@ std::shared_ptr<typename DeciderVerificationKeys::DeciderVK> ProtogalaxyRecursiv
     const FF perturbator_evaluation = evaluate_perturbator(perturbator_coeffs, perturbator_challenge);
 
     std::array<FF, COMBINER_LENGTH>
-        combiner_quotient_evals; // The degree of the combiner quotient (K in the paper) is dk - k - 1 = k(d - 1) - 1.
-                                 // Hence we need  k(d - 1) evaluations to represent it.
+        combiner_quotient_evals; // The degree of the combiner quotient (K in the paper) is dk - k - 1 = k(d - 1)
+                                 // - 1. Hence we need  k(d - 1) evaluations to represent it.
     for (size_t idx = 0; idx < COMBINER_LENGTH; idx++) {
         combiner_quotient_evals[idx] =
             transcript->template receive_from_prover<FF>("combiner_quotient_" + std::to_string(idx + NUM_KEYS));
@@ -98,13 +104,13 @@ std::shared_ptr<typename DeciderVerificationKeys::DeciderVK> ProtogalaxyRecursiv
         (1 - combiner_challenge).[A] + combiner_challenge.[B] == [C]
 
 
-        This reduces the relation to 3 large MSMs where each commitment requires 3 size-128bit scalar multiplications
-        For a flavor with 53 instance/witness commitments, this is 53 * 24 rows
+        This reduces the relation to 3 large MSMs where each commitment requires 3 size-128bit scalar
+       multiplications For a flavor with 53 instance/witness commitments, this is 53 * 24 rows
 
-        Note: there are more efficient ways to evaluate this relationship if one solely wants to reduce number of scalar
-       muls, however we must also consider the number of ECCVM operations being executed, as each operation incurs a
-       cost in the translator circuit Each ECCVM opcode produces 5 rows in the translator circuit, which is approx.
-       equivalent to 9 ECCVM rows. Something to pay attention to
+        Note: there are more efficient ways to evaluate this relationship if one solely wants to reduce number of
+       scalar muls, however we must also consider the number of ECCVM operations being executed, as each operation
+       incurs a cost in the translator circuit Each ECCVM opcode produces 5 rows in the translator circuit, which is
+       approx. equivalent to 9 ECCVM rows. Something to pay attention to
     */
 
     // New transcript for challenge generation