feat: merge-train/barretenberg (#16729)

BEGIN_COMMIT_OVERRIDE
chore: audit ECCVM msm relation (#16532)
feat: add storage budget control for FileBackedMemory (#16690)
fix(bb): fix honk_solidity_key_gen debug build (#16744)
chore: wasm fix (#16751)
END_COMMIT_OVERRIDE

Author: AztecBot

barretenberg/cpp/src/barretenberg/api/api.hpp

@@ -22,6 +22,7 @@ class API {
         bool write_vk{ false };    // should we addditionally write the verification key when writing the proof
         bool include_gates_per_opcode{ false }; // should we include gates_per_opcode in the gates command output
         bool slow_low_memory{ false };          // use file backed memory for polynomials
+        std::string storage_budget;             // storage budget for file backed memory (e.g. "500m", "2g")
         bool update_inputs{ false };            // update inputs when check fails
 
         bool optimized_solidity_verifier{ false }; // should we use the optimized sol verifier? (temp)
@@ -40,6 +41,7 @@ class API {
                << "  write_vk " << flags.write_vk << "\n"
                << "  include_gates_per_opcode " << flags.include_gates_per_opcode << "\n"
                << "  slow_low_memory " << flags.slow_low_memory << "\n"
+               << "  storage_budget " << flags.storage_budget << "\n"
                << "]" << std::endl;
             return os;
         }

barretenberg/cpp/src/barretenberg/bb/cli.cpp

@@ -265,6 +265,13 @@ int parse_and_run_cli_command(int argc, char* argv[])
             "--slow_low_memory", flags.slow_low_memory, "Enable low memory mode (can be 2x slower or more).");
     };
 
+    const auto add_storage_budget_option = [&](CLI::App* subcommand) {
+        return subcommand->add_option("--storage_budget",
+                                      flags.storage_budget,
+                                      "Storage budget for FileBackedMemory (e.g. '500m', '2g'). When exceeded, falls "
+                                      "back to RAM (requires --slow_low_memory).");
+    };
+
     const auto add_update_inputs_flag = [&](CLI::App* subcommand) {
         return subcommand->add_flag("--update_inputs", flags.update_inputs, "Update inputs if vk check fails.");
     };
@@ -345,6 +352,7 @@ int parse_and_run_cli_command(int argc, char* argv[])
     add_slow_low_memory_flag(prove);
     add_print_bench_flag(prove);
     add_bench_out_option(prove);
+    add_storage_budget_option(prove);
 
     prove->add_flag("--verify", "Verify the proof natively, resulting in a boolean output. Useful for testing.");
 
@@ -551,6 +559,9 @@ int parse_and_run_cli_command(int argc, char* argv[])
     verbose_logging = debug_logging || flags.verbose;
     slow_low_memory = flags.slow_low_memory;
 #ifndef __wasm__
+    if (!flags.storage_budget.empty()) {
+        storage_budget = parse_size_string(flags.storage_budget);
+    }
     if (print_bench || !bench_out.empty()) {
         bb::detail::use_bb_bench = true;
     }

barretenberg/cpp/src/barretenberg/benchmark/decrypt_bench/CMakeLists.txt

@@ -1,4 +1,4 @@
-if (NOT FUZZING) 
+if (NOT FUZZING)
     add_executable(
         decrypt_bench
         main.cpp
@@ -10,4 +10,16 @@ if (NOT FUZZING)
         ecc
         common
     )
-endif()
+  if(CHECK_CIRCUIT_STACKTRACES OR ENABLE_STACKTRACES)
+    target_link_libraries(
+        decrypt_bench
+        PUBLIC
+        Backward::Interface
+    )
+    target_link_options(
+        decrypt_bench
+        PRIVATE
+        -ldw -lelf
+    )
+  endif()
+endif()

barretenberg/cpp/src/barretenberg/eccvm/README.md

@@ -28,7 +28,7 @@ template <typename CycleGroup> struct ScalarMul {
     typename CycleGroup::affine_element base_point;
     std::array<int, NUM_WNAF_DIGITS_PER_SCALAR>
         wnaf_digits; // [a_{n-1}, a_{n-1}, ..., a_{0}], where each a_i ∈ {-2ʷ⁻¹ + 1, -2ʷ⁻¹ + 3, ..., 2ʷ⁻¹ - 3, 2ʷ⁻¹ -
-                     // 1} ∪ {0}. (here, w = `NUM_WNAF_DIGIT_BITS`). in particular, a_i is an odd integer with
+                     // 1}. (here, w = `NUM_WNAF_DIGIT_BITS`). in particular, a_i is an odd integer with
                      // absolute value less than 2ʷ. Represents the number `scalar` = ∑ᵢ aᵢ 2⁴ⁱ - `wnaf_skew`.
     bool wnaf_skew;  // necessary to represent _even_ integers
     // size bumped by 1 to record base_point.dbl()

barretenberg/cpp/src/barretenberg/eccvm/eccvm_builder_types.hpp

@@ -182,7 +182,7 @@ class ECCVMCircuitBuilder {
             msm.resize(msm_sizes[i]);
         }
         // populate result using the auxiliary vectors `msm_opqueue_index` and `msm_mul_index`, together with
-        // `eccvm_ops`. this first pass will *not* get the pc (program counter) correct. we explain why when we set it
+        // `eccvm_ops`. this first pass will *not* get the pc (point counter) correct. we explain why when we set it
         // correctly.
         parallel_for_range(msm_opqueue_index.size(), [&](size_t start, size_t end) {
             for (size_t i = start; i < end; i++) {

barretenberg/cpp/src/barretenberg/eccvm/eccvm_circuit_builder.hpp

@@ -549,6 +549,10 @@ class ECCVMFlavor {
          *          msm_lambda2: temp variable used for ecc point addition algorithm if msm_add2 = 1
          *          msm_lambda3: temp variable used for ecc point addition algorithm if msm_add3 = 1
          *          msm_lambda4: temp variable used for ecc point addition algorithm if msm_add4 = 1
+         *          msm_slice1: wNAF digit/slice for first add
+         *          msm_slice2: wNAF digit/slice for second add
+         *          msm_slice3: wNAF digit/slice for third add
+         *          msm_slice4: wNAF digit/slice for fourth add
          *          msm_collision_x1: used to ensure incomplete ecc addition exceptions not triggered if msm_add1 = 1
          *          msm_collision_x2: used to ensure incomplete ecc addition exceptions not triggered if msm_add2 = 1
          *          msm_collision_x3: used to ensure incomplete ecc addition exceptions not triggered if msm_add3 = 1

barretenberg/cpp/src/barretenberg/eccvm/eccvm_flavor.hpp

@@ -27,9 +27,15 @@ class ECCVMMSMMBuilder {
     static constexpr size_t NUM_WNAF_DIGITS_PER_SCALAR = bb::eccvm::NUM_WNAF_DIGITS_PER_SCALAR;
 
     struct alignas(64) MSMRow {
-        uint32_t pc = 0;        // counter over all half-length (128 bit) scalar muls used to compute the required MSMs
-        uint32_t msm_size = 0;  // the number of points that will be scaled and summed
-        uint32_t msm_count = 0; // number of multiplications processed so far in current MSM round
+        uint32_t pc = 0;       // decreasing point-counter, over all half-length (128 bit) scalar muls used to compute
+                               // the required MSMs. however, this value is _constant_ on a given MSM and more precisely
+                               //  refers to the number of half-length scalar muls completed up until we have started
+                               // the current MSM.
+        uint32_t msm_size = 0; // the number of points in (a.k.a. the length of) the MSM in whose computation
+                               // this VM row participates
+        uint32_t msm_count = 0; // number of multiplications processed so far (not including this row) in current MSM
+                                // round (a.k.a. wNAF digit slot). this specifically refers to the number of wNAF-digit
+                                // * point scalar products we have looked up and accumulated.
         uint32_t msm_round = 0; // current "round" of MSM, in {0, ..., 32 = `NUM_WNAF_DIGITS_PER_SCALAR`}. With the
                                 // Straus algorithm, we proceed wNAF digit by wNAF digit, from left to right. (final
                                 // round deals with the `skew` bit.)
@@ -73,7 +79,7 @@ class ECCVMMSMMBuilder {
      * @brief Computes the row values for the Straus MSM columns of the ECCVM.
      *
      * For a detailed description of the Straus algorithm and its relation to the ECCVM, please see
-     * https://hackmd.io/@aztec-network/rJ5xhuCsn
+     * https://hackmd.io/@aztec-network/rJ5xhuCsn or, alternatively, the [ECCVM readme](README.md).
      *
      * @param msms A vector of vectors of `ScalarMul`s, a.k.a. a vector of `MSM`s.
      * @param point_table_read_counts Table of read counts to be populated.
@@ -96,17 +102,23 @@ class ECCVMMSMMBuilder {
         //   row      = point_idx * rows_per_point_table + (some function of the slice value)
         //
         // Illustration:
-        //   Block Structure   Table structure:
-        //      | 0 | 1 |        | Block_{0}                      | <-- pc = total_number_of_muls
-        //      | - | - |        | Block_{1}                      | <-- pc = total_number_of_muls-(num muls in msm 0)
-        //    1 | # | # | -1     |   ...                          | ...
-        //    3 | # | # | -3     | Block_{total_number_of_muls-1} | <-- pc = num muls in last msm
+        //   Block Structure:
+        //      | 0 | 1 |
+        //      | - | - |
+        //    1 | # | # | -1
+        //    3 | # | # | -3
         //    5 | # | # | -5
         //    7 | # | # | -7
         //    9 | # | # | -9
         //   11 | # | # | -11
         //   13 | # | # | -13
         //   15 | # | # | -15
+        //
+        //   Table structure:
+        //    | Block_{0}                      | <-- pc = total_number_of_muls
+        //    | Block_{1}                      | <-- pc = total_number_of_muls-(num muls in msm 0)
+        //    |   ...                          | ...
+        //    | Block_{total_number_of_muls-1} | <-- pc = num muls in last msm
 
         const size_t num_rows_in_read_counts_table =
             static_cast<size_t>(total_number_of_muls) *
@@ -146,7 +158,7 @@ class ECCVMMSMMBuilder {
         std::vector<size_t> msm_row_counts;
         msm_row_counts.reserve(msms.size() + 1);
         msm_row_counts.push_back(1);
-        // compute the program counter (i.e. the index among all single scalar muls) that each multiscalar
+        // compute the point counter (i.e. the index among all single scalar muls) that each multiscalar
         // multiplication will start at.
         std::vector<size_t> pc_values;
         pc_values.reserve(msms.size() + 1);
@@ -203,7 +215,7 @@ class ECCVMMSMMBuilder {
                             bool add = num_points_in_row > relative_point_idx;
                             const size_t point_idx = offset + relative_point_idx;
                             if (add) {
-                                // pc starts at total_number_of_muls and decreases non-uniformly to 0
+                                // `pc` starts at total_number_of_muls and decreases non-uniformly to 0.
                                 // -15 maps to the 1st point in the lookup table (array element 0)
                                 // -1 maps to the point in the lookup table that corresponds to the negation of the
                                 // original input point (i.e. the point we need to add into the accumulator if wnaf_skew

barretenberg/cpp/src/barretenberg/eccvm/msm_builder.hpp

@@ -109,8 +109,8 @@ class ECCVMTranscriptBuilder {
 
     // maintains the state of the VM at any given "time" (i.e., at any given value of pc).
     struct VMState {
-        uint32_t pc = 0; // decreasing program counter that tracks the total number of multiplications that our virtual
-                         // machine has left to compute.
+        uint32_t pc = 0;    // decreasing point counter that tracks the total number of multiplications that our virtual
+                            // machine has left to compute.
         uint32_t count = 0; // Number of muls in the current MSM _excluding the current row_.
         Element accumulator = CycleGroup::affine_point_at_infinity; // accumulator for all group operations.
         Element msm_accumulator =

barretenberg/cpp/src/barretenberg/eccvm/transcript_builder.hpp

@@ -1,5 +1,78 @@
 #include "barretenberg/polynomials/backing_memory.hpp"
+#include "barretenberg/common/throw_or_abort.hpp"
+#include <atomic>
+#include <cctype>
+#include <cstdlib>
+#include <limits>
+#include <string>
 
 // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
 bool slow_low_memory =
     std::getenv("BB_SLOW_LOW_MEMORY") == nullptr ? false : std::string(std::getenv("BB_SLOW_LOW_MEMORY")) == "1";
+
+// Storage budget is disabled for WASM builds
+#ifndef __wasm__
+
+// Parse storage size string (e.g., "500m", "2g", "1024k")
+size_t parse_size_string(const std::string& size_str)
+{
+    if (size_str.empty()) {
+        return std::numeric_limits<size_t>::max();
+    }
+
+    try {
+        std::string str = size_str;
+
+        // Convert to lowercase for case-insensitive comparison
+        char suffix = static_cast<char>(std::tolower(static_cast<unsigned char>(str.back())));
+        size_t multiplier = 1;
+
+        // Check for unit suffix
+        if (suffix == 'k') {
+            multiplier = 1024ULL;
+            str.pop_back();
+        } else if (suffix == 'm') {
+            multiplier = 1024ULL * 1024ULL;
+            str.pop_back();
+        } else if (suffix == 'g') {
+            multiplier = 1024ULL * 1024ULL * 1024ULL;
+            str.pop_back();
+        } else if (std::isdigit(static_cast<unsigned char>(suffix)) == 0) {
+            // Invalid suffix
+            throw_or_abort("Invalid storage size format: '" + size_str + "'. Use format like '500m', '2g', or '1024k'");
+        }
+
+        // Check if remaining string is a valid number
+        if (str.empty()) {
+            throw_or_abort("Invalid storage size format: '" + size_str + "'. No numeric value provided");
+        }
+
+        size_t value = std::stoull(str);
+        return value * multiplier;
+    } catch (const std::invalid_argument&) {
+        throw_or_abort("Invalid storage size format: '" + size_str + "'. Not a valid number");
+    } catch (const std::out_of_range&) {
+        throw_or_abort("Invalid storage size format: '" + size_str + "'. Value out of range");
+    }
+}
+
+namespace {
+// Parse storage budget from environment variable (supports k/m/g suffixes like Docker)
+size_t parse_storage_budget()
+{
+    const char* env_val = std::getenv("BB_STORAGE_BUDGET");
+    if (env_val == nullptr) {
+        return std::numeric_limits<size_t>::max(); // No limit by default
+    }
+
+    return parse_size_string(std::string(env_val));
+}
+} // namespace
+
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+size_t storage_budget = parse_storage_budget();
+
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
+std::atomic<size_t> current_storage_usage{ 0 };
+
+#endif // __wasm__