perf: reduce barretenberg compile overhead

Author: ludamad

barretenberg/cpp/bootstrap.sh

@@ -123,21 +123,37 @@ function write_gtest_inventory {
   local inventory=$build_dir/.aztec-gtest-tests
   local tmp=$inventory.tmp
   local ctest_file
-  local ctest_base
-  local bin_name
+  local ctest_files=()
 
   [ -d "$build_dir" ] || return
-  : > "$tmp"
-  while IFS='|' read -r bin_name ctest_file; do
-    list_tests_from_ctest "$ctest_file" | while read -r test; do
-      printf '%s\t%s\n' "$bin_name" "$test"
-    done
-  done < <(
-    while IFS= read -r ctest_file; do
-      ctest_base=${ctest_file##*/}
-      printf '%s|%s\n' "${ctest_base%%\[*}" "$ctest_file"
-    done < <(find "$build_dir" -type f -name '*_tests[[]*[]]_tests.cmake') | sort
-  ) >> "$tmp"
+  while IFS= read -r -d '' ctest_file; do
+    ctest_files+=("$ctest_file")
+  done < <(find "$build_dir" -type f -name '*_tests[[]*[]]_tests.cmake' -print0 | sort -z)
+
+  if [ "${#ctest_files[@]}" -gt 0 ]; then
+    awk '
+      FNR == 1 {
+        bin = FILENAME
+        sub(/^.*\//, "", bin)
+        sub(/\[.*/, "", bin)
+      }
+      /--gtest_filter=/ {
+        line = $0
+        while (match(line, /--gtest_filter=/)) {
+          line = substr(line, RSTART + RLENGTH)
+          test = line
+          sub(/\].*/, "", test)
+          sub(/[" )].*/, "", test)
+          if (test != "" && test !~ /DISABLED_/) {
+            print bin "\t" test
+          }
+          line = substr(line, length(test) + 1)
+        }
+      }
+    ' "${ctest_files[@]}" > "$tmp"
+  else
+    : > "$tmp"
+  fi
 
   if [ -s "$tmp" ]; then
     mv "$tmp" "$inventory"
@@ -314,61 +330,67 @@ function build {
   fi
 }
 
+function emit_native_test_cmd {
+  local bin_name=$1
+  local test=$2
+
+  # Skip heavy recursion tests in debug builds — they take 400-600s+ and the same
+  # code paths are already exercised (with assertions) by faster tests in the suite.
+  # Keep WithoutPredicate/1.GenerateVKFromConstraints (241s) so that the debug-only
+  # native_verification_debug path in honk_recursion_constraint.cpp is still exercised.
+  # None of the other skipped suites exercise unique debug-only (#ifndef NDEBUG) code paths.
+  if [[ "$native_preset" == *debug* ]] && [[ "$test" =~ ^(HonkRecursionConstraintTest|ChonkRecursionConstraintTest|AvmRecursionInnerCircuitTests|AvmRecursionConstraintTest|AvmRecursiveTests\.TwoLayer|PaddingVariants/AvmRecursiveTestsParameterized\.TwoLayer|BoomerangTwoLayerAvmRecursiveVerifierTests|ECCVMRecursiveTests|GoblinRecursiveVerifierTests|GoblinAvmRecursiveVerifierTests|BoomerangGoblinRecursiveVerifierTests|BoomerangGoblinAvmRecursiveVerifierTests) ]]; then
+    if [[ "$test" != "HonkRecursionConstraintTestWithoutPredicate/1.GenerateVKFromConstraints" ]]; then
+      return
+    fi
+  fi
+
+  local prefix=$hash
+  # A little extra resource for these tests.
+  # IPARecursiveTests fails with 2 threads.
+  if [[ "$test" =~ ^(AcirAvmRecursionConstraint|ChonkKernelCapacity|AvmRecursiveTests|IPARecursiveTests|HonkRecursionConstraintTest|ChonkRecursionConstraintTest) ]]; then
+    prefix="$prefix:CPUS=4:MEM=8g"
+  fi
+  echo -e "$prefix barretenberg/cpp/scripts/run_test.sh $bin_name $test"
+}
+
 function test_cmds_native {
   # E.g. build, build-debug or build-coverage
   cd $native_build_dir
 
-  declare -A inventory_tests=()
-  local inv_bin
-  local inv_test
-  if [ -f .aztec-gtest-tests ]; then
-    while IFS=$'\t' read -r inv_bin inv_test; do
-      inventory_tests["$inv_bin"]+="$inv_test"$'\n'
+  local bin_name
+  local test
+  if [ -s .aztec-gtest-tests ]; then
+    while IFS=$'\t' read -r bin_name test; do
+      [ -n "$bin_name" ] && [ -n "$test" ] || continue
+      emit_native_test_cmd "$bin_name" "$test"
     done < .aztec-gtest-tests
-  fi
-
-  declare -A discovered_tests=()
-  local ctest_file
-  local ctest_base
-  while IFS= read -r ctest_file; do
-    ctest_base=${ctest_file##*/}
-    discovered_tests["${ctest_base%%\[*}"]=$ctest_file
-  done < <(find . -type f -name '*_tests[[]*[]]_tests.cmake')
-
-  for bin in ./bin/*_tests; do
-    local bin_name=${bin##*/}
-
-    {
-      if [ -n "${inventory_tests[$bin_name]:-}" ]; then
-        printf '%s' "${inventory_tests[$bin_name]}"
-      elif [ -n "${discovered_tests[$bin_name]:-}" ]; then
-        list_tests_from_ctest "${discovered_tests[$bin_name]}"
-      else
-        $bin --gtest_list_tests | \
-          awk '/^[a-zA-Z]/ {suite=$1} /^[ ]/ {print suite$1}' | \
-          grep -v 'DISABLED_'
-      fi
-    } | \
-      while read -r test; do
-        # Skip heavy recursion tests in debug builds — they take 400-600s+ and the same
-        # code paths are already exercised (with assertions) by faster tests in the suite.
-        # Keep WithoutPredicate/1.GenerateVKFromConstraints (241s) so that the debug-only
-        # native_verification_debug path in honk_recursion_constraint.cpp is still exercised.
-        # None of the other skipped suites exercise unique debug-only (#ifndef NDEBUG) code paths.
-        if [[ "$native_preset" == *debug* ]] && [[ "$test" =~ ^(HonkRecursionConstraintTest|ChonkRecursionConstraintTest|AvmRecursionInnerCircuitTests|AvmRecursionConstraintTest|AvmRecursiveTests\.TwoLayer|PaddingVariants/AvmRecursiveTestsParameterized\.TwoLayer|BoomerangTwoLayerAvmRecursiveVerifierTests|ECCVMRecursiveTests|GoblinRecursiveVerifierTests|GoblinAvmRecursiveVerifierTests|BoomerangGoblinRecursiveVerifierTests|BoomerangGoblinAvmRecursiveVerifierTests) ]]; then
-          if [[ "$test" != "HonkRecursionConstraintTestWithoutPredicate/1.GenerateVKFromConstraints" ]]; then
-            continue
-          fi
-        fi
-        local prefix=$hash
-        # A little extra resource for these tests.
-        # IPARecursiveTests fails with 2 threads.
-        if [[ "$test" =~ ^(AcirAvmRecursionConstraint|ChonkKernelCapacity|AvmRecursiveTests|IPARecursiveTests|HonkRecursionConstraintTest|ChonkRecursionConstraintTest) ]]; then
-          prefix="$prefix:CPUS=4:MEM=8g"
+  else
+    declare -A discovered_tests=()
+    local bin
+    local ctest_file
+    local ctest_base
+    while IFS= read -r ctest_file; do
+      ctest_base=${ctest_file##*/}
+      discovered_tests["${ctest_base%%\[*}"]=$ctest_file
+    done < <(find . -type f -name '*_tests[[]*[]]_tests.cmake')
+
+    for bin in ./bin/*_tests; do
+      bin_name=${bin##*/}
+
+      {
+        if [ -n "${discovered_tests[$bin_name]:-}" ]; then
+          list_tests_from_ctest "${discovered_tests[$bin_name]}"
+        else
+          $bin --gtest_list_tests | \
+            awk '/^[a-zA-Z]/ {suite=$1} /^[ ]/ {print suite$1}' | \
+            grep -v 'DISABLED_'
         fi
-        echo -e "$prefix barretenberg/cpp/scripts/run_test.sh $bin_name $test"
+      } | while read -r test; do
+        emit_native_test_cmd "$bin_name" "$test"
       done || (echo "Failed to list tests in $bin" && exit 1)
-  done
+    done
+  fi
 
   # The pinned IVC inputs / VKs live in the public repo; the private fork
   # carries divergent circuits so the check is expected to fail there.

barretenberg/cpp/src/barretenberg/bbapi/bbapi_chonk.cpp

@@ -47,7 +47,8 @@ ChonkLoad::Response ChonkLoad::execute(BBApiRequest& request) &&
     }
 
     request.loaded_circuit_name = circuit.name;
-    request.loaded_circuit_constraints = acir_format::circuit_buf_to_acir_format(std::move(circuit.bytecode));
+    request.loaded_circuit_constraints =
+        std::make_shared<acir_format::AcirFormat>(acir_format::circuit_buf_to_acir_format(std::move(circuit.bytecode)));
     request.loaded_circuit_vk = circuit.verification_key;
 
     info("ChonkLoad - loaded circuit '", request.loaded_circuit_name, "'");
@@ -62,16 +63,15 @@ ChonkAccumulate::Response ChonkAccumulate::execute(BBApiRequest& request) &&
         throw_or_abort("Chonk not started. Call ChonkStart first.");
     }
 
-    if (!request.loaded_circuit_constraints.has_value()) {
+    if (!request.loaded_circuit_constraints) {
         throw_or_abort("No circuit loaded. Call ChonkLoad first.");
     }
 
     acir_format::WitnessVector witness_data = acir_format::witness_buf_to_witness_vector(std::move(witness));
-    acir_format::AcirProgram program{ std::move(request.loaded_circuit_constraints.value()), std::move(witness_data) };
+    acir_format::AcirProgram program{ std::move(*request.loaded_circuit_constraints), std::move(witness_data) };
 
     // Clear loaded state immediately after moving out of it. This ensures that if any subsequent
-    // step throws, the request won't appear to still have a valid circuit loaded (the optional
-    // would be in a moved-from state, which is technically has_value()==true but poisoned).
+    // step throws, the request won't appear to still have a valid circuit loaded.
     auto loaded_vk = std::move(request.loaded_circuit_vk);
     auto circuit_name = std::move(request.loaded_circuit_name);
     request.loaded_circuit_constraints.reset();

barretenberg/cpp/src/barretenberg/bbapi/bbapi_chonk.hpp

@@ -8,15 +8,14 @@
  * and the batch verifier service (start/queue/stop lifecycle).
  */
 #include "barretenberg/bbapi/bbapi_shared.hpp"
-#include "barretenberg/chonk/chonk.hpp"
+#include "barretenberg/chonk/chonk_proof.hpp"
 #include "barretenberg/common/named_union.hpp"
 #include "barretenberg/honk/proof_system/types/proof.hpp"
 #include "barretenberg/serialize/msgpack.hpp"
 
 #ifndef __wasm__
 #include "barretenberg/chonk/batch_verifier_types.hpp"
 #include "barretenberg/chonk/chonk_batch_verifier.hpp"
-#include "barretenberg/chonk/chonk_proof.hpp"
 #include <condition_variable>
 #include <mutex>
 #include <queue>

barretenberg/cpp/src/barretenberg/bbapi/bbapi_execute.cpp

@@ -1,6 +1,24 @@
 #include "bbapi_execute.hpp"
 
 namespace bb::bbapi {
+CommandResponse execute(BBApiRequest& request, Command&& command)
+{
+    // Reset error state before execution
+    request.error_message.clear();
+
+    CommandResponse response = std::move(command).visit([&request](auto&& cmd) -> CommandResponse {
+        using CmdType = std::decay_t<decltype(cmd)>;
+        return std::forward<CmdType>(cmd).execute(request);
+    });
+
+    // Check if an error occurred during execution
+    if (!request.error_message.empty()) {
+        return ErrorResponse{ .message = std::move(request.error_message) };
+    }
+
+    return response;
+}
+
 namespace { // anonymous
 struct Api {
     Command commands;

barretenberg/cpp/src/barretenberg/bbapi/bbapi_execute.hpp

@@ -5,8 +5,8 @@
 #include "barretenberg/bbapi/bbapi_crypto.hpp"
 #include "barretenberg/bbapi/bbapi_ecc.hpp"
 #include "barretenberg/bbapi/bbapi_ecdsa.hpp"
-#include "barretenberg/bbapi/bbapi_schnorr.hpp"
 #include "barretenberg/bbapi/bbapi_schema.hpp"
+#include "barretenberg/bbapi/bbapi_schnorr.hpp"
 #include "barretenberg/bbapi/bbapi_shared.hpp"
 #include "barretenberg/bbapi/bbapi_srs.hpp"
 #include "barretenberg/bbapi/bbapi_ultra_honk.hpp"
@@ -149,22 +149,6 @@ using CommandResponse = NamedUnion<ErrorResponse,
  * @param request The circuit registry (acting as the request context).
  * @return A variant of all possible command responses.
  */
-inline CommandResponse execute(BBApiRequest& request, Command&& command)
-{
-    // Reset error state before execution
-    request.error_message.clear();
-
-    CommandResponse response = std::move(command).visit([&request](auto&& cmd) -> CommandResponse {
-        using CmdType = std::decay_t<decltype(cmd)>;
-        return std::forward<CmdType>(cmd).execute(request);
-    });
-
-    // Check if an error occurred during execution
-    if (!request.error_message.empty()) {
-        return ErrorResponse{ .message = std::move(request.error_message) };
-    }
-
-    return response;
-}
+CommandResponse execute(BBApiRequest& request, Command&& command);
 
 } // namespace bb::bbapi

barretenberg/cpp/src/barretenberg/bbapi/bbapi_shared.hpp

@@ -7,24 +7,32 @@
  * including circuit input types and proof system settings.
  */
 
-#include "barretenberg/chonk/chonk.hpp"
 #include "barretenberg/common/throw_or_abort.hpp"
-#include "barretenberg/dsl/acir_format/acir_format.hpp"
 #include "barretenberg/flavor/ultra_flavor.hpp"
 #include "barretenberg/flavor/ultra_keccak_flavor.hpp"
 #include "barretenberg/flavor/ultra_keccak_zk_flavor.hpp"
 #include "barretenberg/flavor/ultra_zk_flavor.hpp"
 #include "barretenberg/honk/execution_trace/mega_execution_trace.hpp"
-#include "barretenberg/stdlib/special_public_inputs/special_public_inputs.hpp"
+#include "barretenberg/special_public_inputs/special_public_inputs.hpp"
 #include "barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp"
 #ifdef STARKNET_GARAGA_FLAVORS
 #include "barretenberg/flavor/ultra_starknet_flavor.hpp"
 #include "barretenberg/flavor/ultra_starknet_zk_flavor.hpp"
 #endif
 #include <cstdint>
+#include <memory>
 #include <string>
+#include <type_traits>
 #include <vector>
 
+namespace acir_format {
+struct AcirFormat;
+}
+
+namespace bb {
+class IVCBase;
+}
+
 namespace bb::bbapi {
 
 /**
@@ -182,7 +190,7 @@ struct BBApiRequest {
     // Name of the last loaded circuit
     std::string loaded_circuit_name;
     // Store the parsed constraint system to get ahead of parsing before accumulate
-    std::optional<acir_format::AcirFormat> loaded_circuit_constraints;
+    std::shared_ptr<acir_format::AcirFormat> loaded_circuit_constraints;
     // Store the verification key passed with the circuit
     std::vector<uint8_t> loaded_circuit_vk;
     // Policy for handling verification keys during accumulation

barretenberg/cpp/src/barretenberg/circuit_checker/circuit_checker.hpp

@@ -1,7 +1,5 @@
 #pragma once
-#include "barretenberg/circuit_checker/ultra_circuit_checker.hpp"
-#include "barretenberg/stdlib_circuit_builders/mega_circuit_builder.hpp"
-#include "barretenberg/stdlib_circuit_builders/ultra_circuit_builder.hpp"
+#include "barretenberg/stdlib/primitives/circuit_builders/circuit_builders_fwd.hpp"
 
 namespace bb {
 
@@ -11,8 +9,8 @@ namespace bb {
  */
 class CircuitChecker {
   public:
-    static bool check(const UltraCircuitBuilder& builder) { return UltraCircuitChecker::check(builder); }
-    static bool check(const MegaCircuitBuilder& builder) { return UltraCircuitChecker::check(builder); }
+    static bool check(const UltraCircuitBuilder& builder);
+    static bool check(const MegaCircuitBuilder& builder);
 };
 
 } // namespace bb

barretenberg/cpp/src/barretenberg/circuit_checker/ultra_circuit_checker.cpp

@@ -1,11 +1,22 @@
 #include "ultra_circuit_checker.hpp"
+#include "barretenberg/circuit_checker/circuit_checker.hpp"
 #include "barretenberg/common/assert.hpp"
 #include "barretenberg/flavor/mega_flavor.hpp"
 #include "barretenberg/stdlib/primitives/circuit_builders/circuit_builders.hpp"
 #include <unordered_set>
 
 namespace bb {
 
+bool CircuitChecker::check(const UltraCircuitBuilder& builder)
+{
+    return UltraCircuitChecker::check(builder);
+}
+
+bool CircuitChecker::check(const MegaCircuitBuilder& builder)
+{
+    return UltraCircuitChecker::check(builder);
+}
+
 template <> auto UltraCircuitChecker::init_empty_values<UltraCircuitBuilder_<UltraExecutionTraceBlocks>>()
 {
     return UltraFlavor::AllValues{};

barretenberg/cpp/src/barretenberg/ultra_honk/mega_honk.test.cpp

@@ -2,7 +2,6 @@
 #include <cstdint>
 #include <gtest/gtest.h>
 
-#include "barretenberg/circuit_checker/circuit_checker.hpp"
 #include "barretenberg/common/log.hpp"
 #include "barretenberg/goblin/mock_circuits.hpp"
 #include "barretenberg/honk/proof_length.hpp"

barretenberg/cpp/src/barretenberg/ultra_honk/permutation.test.cpp

@@ -1,4 +1,3 @@
-#include "barretenberg/circuit_checker/circuit_checker.hpp"
 #include "barretenberg/honk/library/grand_product_delta.hpp"
 #include "barretenberg/honk/library/grand_product_library.hpp"
 #include "barretenberg/honk/relation_checker.hpp"