diff --git a/Makefile b/Makefile index 3f65d4bc9d29..81e3380a5df7 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ endef # PHONY TARGETS - List every target that has a file/dir of the same name. #============================================================================== -.PHONY: noir barretenberg noir-projects l1-contracts release-image boxes playground docs aztec-up spartan wsdb bb-avm-sim +.PHONY: noir barretenberg noir-projects l1-contracts release-image boxes playground docs aztec-up spartan lmdblib kvdb wsdb bb-avm-sim #============================================================================== # BOOTSTRAP TARGETS @@ -55,7 +55,7 @@ endef # Fast bootstrap. fast: release-image barretenberg boxes playground docs aztec-up \ - bb-tests l1-contracts-tests yarn-project-tests boxes-tests playground-tests aztec-up-tests docs-tests noir-protocol-circuits-tests contract-snapshots-tests release-image-tests spartan claude-tests ipc-codegen-tests + bb-tests l1-contracts-tests yarn-project-tests boxes-tests playground-tests aztec-up-tests docs-tests noir-protocol-circuits-tests contract-snapshots-tests release-image-tests spartan claude-tests ipc-codegen-tests lmdblib-tests wsdb-tests # Full bootstrap. full: fast bb-full-tests bb-cpp-full yarn-project-benches @@ -326,11 +326,29 @@ ipc-runtime-cross-arm64-macos: ipc-runtime-cross: ipc-runtime ipc-runtime-cross-arm64-linux ipc-runtime-cross-amd64-macos ipc-runtime-cross-arm64-macos #============================================================================== -# WSDB +# Native packages (lmdblib, kvdb, wsdb) #============================================================================== -wsdb: ipc-codegen ipc-runtime bb-cpp-native - $(call build,$@,wsdb) +# lmdblib and kvdb are barretenberg-free: they build against their own deps +# (lmdb, msgpack-c, node-addon-api) only, never bb. +.PHONY: lmdblib kvdb lmdblib-tests wsdb-tests +lmdblib: + $(call build,$@,native-packages/lmdblib) + +kvdb: lmdblib + $(call build,$@,native-packages/kvdb) + +wsdb: ipc-codegen ipc-runtime bb-cpp-native lmdblib + $(call build,$@,native-packages/wsdb) + +# Native-package C++ tests (self-contained gtest binaries). kvdb has no C++ tests +# (its NAPI is exercised by yarn-project's kv-store tests). wsdb_tests use no bb +# headers; the bb-header parity/equivalence target is manual (WSDB_BUILD_BB_TESTS). +lmdblib-tests: lmdblib + $(call test,$@,native-packages/lmdblib) + +wsdb-tests: wsdb + $(call test,$@,native-packages/wsdb) #============================================================================== # .claude tooling @@ -421,7 +439,7 @@ l1-contracts-tests: l1-contracts-verifier # Yarn Project - TypeScript monorepo with all TS packages #============================================================================== -yarn-project: bb-ts noir-projects l1-contracts wsdb bb-avm-sim +yarn-project: bb-ts noir-projects l1-contracts wsdb kvdb bb-avm-sim $(call build,$@,yarn-project) yarn-project-tests: yarn-project diff --git a/aztec-up/bootstrap.sh b/aztec-up/bootstrap.sh index 8bf6abb5f9ff..2377ba6ff861 100755 --- a/aztec-up/bootstrap.sh +++ b/aztec-up/bootstrap.sh @@ -1,16 +1,23 @@ #!/usr/bin/env bash source $(git rev-parse --show-toplevel)/ci3/source_bootstrap -hash=$(hash_str $(cache_content_hash ^aztec-up/) $(../ipc-runtime/bootstrap.sh hash) $(../wsdb/bootstrap.sh hash) $(../barretenberg/ts/bootstrap.sh hash) $(../yarn-project/bootstrap.sh hash)) +hash=$(hash_str $(cache_content_hash ^aztec-up/) $(../ipc-runtime/bootstrap.sh hash) $(../native-packages/kvdb/bootstrap.sh hash) $(../native-packages/wsdb/bootstrap.sh hash) $(../barretenberg/ts/bootstrap.sh hash) $(../yarn-project/bootstrap.sh hash)) # Bare aliases ("nightly", "latest") resolve to this major version. DEFAULT_MAJOR_VERSION=${AZTEC_TOOLCHAIN_DEFAULT_MAJOR_VERSION:-4} function wsdb_package_dirs { - for package_dir in "$root"/wsdb/ts/packages/*; do + for package_dir in "$root"/native-packages/wsdb/ts/packages/*; do [ -d "$package_dir" ] && echo "$package_dir" done - echo "$root/wsdb/ts" + echo "$root/native-packages/wsdb/ts" +} + +function kvdb_package_dirs { + for package_dir in "$root"/native-packages/kvdb/ts/packages/*; do + [ -d "$package_dir" ] && echo "$package_dir" + done + echo "$root/native-packages/kvdb/ts" } function barretenberg_ts_package_dirs { @@ -117,6 +124,7 @@ EOF echo $root/ipc-runtime/ts barretenberg_ts_package_dirs wsdb_package_dirs + kvdb_package_dirs # l1-artifacts lives under l1-contracts, so it isn't enumerated by yarn-project's get_projects; # publish it explicitly or @aztec/aztec's portal dependency can't resolve from the local registry. echo $root/l1-contracts/l1-artifacts diff --git a/barretenberg/cpp/CMakeLists.txt b/barretenberg/cpp/CMakeLists.txt index 1c4b2038a76d..0cf6793849b4 100644 --- a/barretenberg/cpp/CMakeLists.txt +++ b/barretenberg/cpp/CMakeLists.txt @@ -12,7 +12,7 @@ project( # === Platform Guard Reference === # __wasm__ : Compiler-defined for WebAssembly. Guards threading, file I/O, networking. # _WIN32 : Compiler-defined for Windows. Guards POSIX APIs (mmap, signals, IPC). -# BB_LITE : CMake option. Excludes server-side subsystems (lmdb, world_state, ipc, nodejs_module). +# BB_LITE : CMake option. Excludes server-side subsystems (ipc, vm2_wsdb, cdb, avm). # Enabled for: iOS, Android, Windows cross-compiles. # WASM : CMake variable mirroring __wasm__ for CMake-level logic. @@ -45,7 +45,7 @@ option(ENABLE_PIC "Builds with position independent code" OFF) option(SYNTAX_ONLY "only check syntax (-fsyntax-only)" OFF) option(ENABLE_WASM_BENCH "Enable BB_BENCH benchmarking support in WASM builds (dev only, not for releases)" OFF) option(AVM "enable building of vm2 module and bb-avm" ON) -option(BB_LITE "Exclude server-side subsystems: lmdb, world_state, ipc, nodejs_module" OFF) +option(BB_LITE "Exclude server-side subsystems: ipc, vm2_wsdb, cdb, avm" OFF) if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") message(STATUS "Compiling for ARM.") diff --git a/barretenberg/cpp/CMakePresets.json b/barretenberg/cpp/CMakePresets.json index 5eac372c31c2..1c75c2ebb68a 100644 --- a/barretenberg/cpp/CMakePresets.json +++ b/barretenberg/cpp/CMakePresets.json @@ -801,25 +801,25 @@ "name": "amd64-linux", "configurePreset": "amd64-linux", "inheritConfigureEnvironment": true, - "targets": ["bb", "nodejs_module", "bb-external", "aztec-wsdb", "bb-avm-sim"] + "targets": ["bb", "nodejs_module", "bb-external", "bb-avm-sim"] }, { "name": "arm64-linux", "configurePreset": "arm64-linux", "inheritConfigureEnvironment": true, - "targets": ["bb", "nodejs_module", "bb-external", "aztec-wsdb", "bb-avm-sim"] + "targets": ["bb", "nodejs_module", "bb-external", "bb-avm-sim"] }, { "name": "amd64-macos", "configurePreset": "amd64-macos", "inheritConfigureEnvironment": true, - "targets": ["bb", "nodejs_module", "bb-external", "aztec-wsdb", "bb-avm-sim"] + "targets": ["bb", "nodejs_module", "bb-external", "bb-avm-sim"] }, { "name": "arm64-macos", "configurePreset": "arm64-macos", "inheritConfigureEnvironment": true, - "targets": ["bb", "nodejs_module", "bb-external", "aztec-wsdb", "bb-avm-sim"] + "targets": ["bb", "nodejs_module", "bb-external", "bb-avm-sim"] }, { "name": "amd64-windows", diff --git a/barretenberg/cpp/scripts/audit/audit_scopes/merkle_tree_audit_scope.md b/barretenberg/cpp/scripts/audit/audit_scopes/merkle_tree_audit_scope.md index 6e4f0e9f5eef..2728ccabb5ef 100644 --- a/barretenberg/cpp/scripts/audit/audit_scopes/merkle_tree_audit_scope.md +++ b/barretenberg/cpp/scripts/audit/audit_scopes/merkle_tree_audit_scope.md @@ -20,14 +20,14 @@ Note: Paths relative to `aztec-packages/barretenberg/cpp/src/barretenberg` ### Node Store (Cache + Metadata) 8. `crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp` 9. `crypto/merkle_tree/node_store/content_addressed_cache.hpp` -10. `crypto/merkle_tree/node_store/tree_meta.hpp` +10. `crypto/merkle_tree/tree_meta.hpp` ### Append-Only Tree 11. `crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp` ### Indexed Tree 12. `crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp` -13. `crypto/merkle_tree/indexed_tree/indexed_leaf.hpp` +13. `crypto/merkle_tree/indexed_leaf.hpp` ## Summary of Module diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt index 7ccf465413e4..1db0dc9cdf5d 100644 --- a/barretenberg/cpp/src/CMakeLists.txt +++ b/barretenberg/cpp/src/CMakeLists.txt @@ -116,22 +116,17 @@ add_subdirectory(barretenberg/ultra_honk) add_subdirectory(barretenberg/vm2_stub) add_subdirectory(barretenberg/wasi) -if(NOT BB_LITE) - add_subdirectory(barretenberg/lmdblib) -endif() - if(NOT FUZZING AND NOT WASM AND NOT BB_LITE) - # Fuzzing preset cannot be built with world_state as world_state cannot compile with MULTITHREADING=OFF - # Mobile builds exclude these modules that require LMDB or aren't needed on mobile + # Mobile builds exclude these modules that aren't needed on mobile add_subdirectory(barretenberg/aztec) - add_subdirectory(barretenberg/world_state) # NOTE: Do not conditionally base this on the AVM flag as it defines a necessary vm2_sim library. add_subdirectory(barretenberg/vm2) add_subdirectory(barretenberg/ipc) - add_subdirectory(barretenberg/wsdb) add_subdirectory(barretenberg/vm2_wsdb) add_subdirectory(barretenberg/cdb) add_subdirectory(barretenberg/avm) + # nodejs_module hosts only the msgpack_client NAPI wrappers now (bb.js's SHM + # transport). The lmdb_store kv-store NAPI moved to native-packages/kvdb. add_subdirectory(barretenberg/nodejs_module) endif() @@ -147,7 +142,6 @@ if(FUZZING_AVM) if(FUZZING) # Only add these if they weren't added above (when NOT FUZZING AND NOT WASM) add_subdirectory(barretenberg/aztec) - add_subdirectory(barretenberg/world_state) add_subdirectory(barretenberg/vm2) endif() add_subdirectory(barretenberg/avm_fuzzer) @@ -216,11 +210,9 @@ set(BARRETENBERG_TARGET_OBJECTS set(BB_EXTERNAL_TARGET_OBJECTS ${BARRETENBERG_TARGET_OBJECTS}) if(NOT WASM AND NOT FUZZING AND NOT BB_LITE) - # enable merkle trees and lmdb (not for mobile builds) + # enable merkle trees (not for mobile builds) list(APPEND BARRETENBERG_TARGET_OBJECTS $) - list(APPEND BARRETENBERG_TARGET_OBJECTS $) list(APPEND BARRETENBERG_TARGET_OBJECTS $) - list(APPEND BARRETENBERG_TARGET_OBJECTS $) endif() add_library( diff --git a/barretenberg/cpp/src/barretenberg/avm/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/avm/CMakeLists.txt index 16610a355d99..76acead0a975 100644 --- a/barretenberg/cpp/src/barretenberg/avm/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/avm/CMakeLists.txt @@ -49,7 +49,6 @@ target_link_libraries( barretenberg vm2_sim wsdb_ipc_merkle_db - wsdb_ipc_client cdb_ipc_client ipc_runtime env diff --git a/barretenberg/cpp/src/barretenberg/avm/avm_execute.cpp b/barretenberg/cpp/src/barretenberg/avm/avm_execute.cpp index 4d00b7ec156a..f10014c7401f 100644 --- a/barretenberg/cpp/src/barretenberg/avm/avm_execute.cpp +++ b/barretenberg/cpp/src/barretenberg/avm/avm_execute.cpp @@ -11,7 +11,7 @@ namespace bb::avm { using namespace bb::avm2; -using namespace bb::world_state; +using bb::crypto::merkle_tree::WorldStateRevision; // Cancellation for the single in-flight simulation. bb-avm-sim runs exactly one // simulation at a time; the SIGUSR1 handler (which may run on any thread) cancels diff --git a/barretenberg/cpp/src/barretenberg/avm/avm_execute.hpp b/barretenberg/cpp/src/barretenberg/avm/avm_execute.hpp index 4f9a18f0b33f..37f2a85dd5c7 100644 --- a/barretenberg/cpp/src/barretenberg/avm/avm_execute.hpp +++ b/barretenberg/cpp/src/barretenberg/avm/avm_execute.hpp @@ -7,7 +7,7 @@ #include "barretenberg/avm/generated/avm_ipc_server.hpp" #include "barretenberg/cdb/cdb_ipc_client.hpp" #include "barretenberg/vm2/simulation/lib/cancellation_token.hpp" -#include "barretenberg/wsdb/generated/wsdb_ipc_client.hpp" +#include "barretenberg/vm2_wsdb/generated/wsdb_ipc_client.hpp" #include diff --git a/barretenberg/cpp/src/barretenberg/avm/avm_ipc_server.cpp b/barretenberg/cpp/src/barretenberg/avm/avm_ipc_server.cpp index 80f3fed08372..4f2443edacdb 100644 --- a/barretenberg/cpp/src/barretenberg/avm/avm_ipc_server.cpp +++ b/barretenberg/cpp/src/barretenberg/avm/avm_ipc_server.cpp @@ -2,7 +2,7 @@ #include "barretenberg/avm/avm_execute.hpp" #include "barretenberg/cdb/cdb_ipc_client.hpp" #include "barretenberg/common/log.hpp" -#include "barretenberg/wsdb/generated/wsdb_ipc_client.hpp" +#include "barretenberg/vm2_wsdb/generated/wsdb_ipc_client.hpp" #include "ipc_runtime/ipc_server.hpp" #include "ipc_runtime/serve_helper.hpp" #include "ipc_runtime/signal_handlers.hpp" diff --git a/barretenberg/cpp/src/barretenberg/avm_fuzzer/common/interfaces/dbs.cpp b/barretenberg/cpp/src/barretenberg/avm_fuzzer/common/interfaces/dbs.cpp index b3ed85e93f2c..8ed58bfe6999 100644 --- a/barretenberg/cpp/src/barretenberg/avm_fuzzer/common/interfaces/dbs.cpp +++ b/barretenberg/cpp/src/barretenberg/avm_fuzzer/common/interfaces/dbs.cpp @@ -6,7 +6,7 @@ #include "barretenberg/avm_fuzzer/fuzz_lib/constants.hpp" #include "barretenberg/aztec/aztec_constants.hpp" #include "barretenberg/common/throw_or_abort.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/crypto/poseidon2/poseidon2.hpp" #include "barretenberg/vm2/common/aztec_types.hpp" #include "barretenberg/vm2/simulation/lib/contract_crypto.hpp" diff --git a/barretenberg/cpp/src/barretenberg/avm_fuzzer/common/interfaces/dbs.hpp b/barretenberg/cpp/src/barretenberg/avm_fuzzer/common/interfaces/dbs.hpp index 5f427789c34a..386cd947261e 100644 --- a/barretenberg/cpp/src/barretenberg/avm_fuzzer/common/interfaces/dbs.hpp +++ b/barretenberg/cpp/src/barretenberg/avm_fuzzer/common/interfaces/dbs.hpp @@ -3,7 +3,7 @@ #include #include -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/vm2/common/aztec_types.hpp" #include "barretenberg/vm2/simulation/interfaces/db.hpp" #include "barretenberg/vm2/simulation/lib/memory_merkle_db.hpp" diff --git a/barretenberg/cpp/src/barretenberg/avm_fuzzer/fuzz_lib/simulator.hpp b/barretenberg/cpp/src/barretenberg/avm_fuzzer/fuzz_lib/simulator.hpp index abd43ba6a452..485d4ea445c2 100644 --- a/barretenberg/cpp/src/barretenberg/avm_fuzzer/fuzz_lib/simulator.hpp +++ b/barretenberg/cpp/src/barretenberg/avm_fuzzer/fuzz_lib/simulator.hpp @@ -3,7 +3,7 @@ #include #include "barretenberg/avm_fuzzer/common/interfaces/dbs.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/vm2/common/avm_io.hpp" #include "barretenberg/vm2/common/aztec_types.hpp" #include "barretenberg/vm2/common/field.hpp" diff --git a/barretenberg/cpp/src/barretenberg/avm_fuzzer/fuzzer_lib.hpp b/barretenberg/cpp/src/barretenberg/avm_fuzzer/fuzzer_lib.hpp index 4033f3aab95d..564e96030737 100644 --- a/barretenberg/cpp/src/barretenberg/avm_fuzzer/fuzzer_lib.hpp +++ b/barretenberg/cpp/src/barretenberg/avm_fuzzer/fuzzer_lib.hpp @@ -11,7 +11,7 @@ #include "barretenberg/avm_fuzzer/fuzz_lib/simulator.hpp" #include "barretenberg/avm_fuzzer/mutations/bytecode.hpp" #include "barretenberg/avm_fuzzer/mutations/tx_data.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/serialize/msgpack_impl.hpp" #include "barretenberg/vm2/common/avm_io.hpp" #include "barretenberg/vm2/common/aztec_types.hpp" diff --git a/barretenberg/cpp/src/barretenberg/avm_fuzzer/mutations/bytecode.hpp b/barretenberg/cpp/src/barretenberg/avm_fuzzer/mutations/bytecode.hpp index 6cb28e9aa3f9..02f56291017d 100644 --- a/barretenberg/cpp/src/barretenberg/avm_fuzzer/mutations/bytecode.hpp +++ b/barretenberg/cpp/src/barretenberg/avm_fuzzer/mutations/bytecode.hpp @@ -1,6 +1,6 @@ #pragma once -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/vm2/common/aztec_types.hpp" #include #include diff --git a/barretenberg/cpp/src/barretenberg/aztec/aztec_hash_policy.hpp b/barretenberg/cpp/src/barretenberg/aztec/aztec_hash_policy.hpp index 7f172ce1a7e7..5ccc0cca10e2 100644 --- a/barretenberg/cpp/src/barretenberg/aztec/aztec_hash_policy.hpp +++ b/barretenberg/cpp/src/barretenberg/aztec/aztec_hash_policy.hpp @@ -2,7 +2,7 @@ #include "barretenberg/aztec/aztec_constants.hpp" #include "barretenberg/crypto/merkle_tree/hash.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" namespace bb::aztec { diff --git a/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt index 2103fa9f89f8..2916043712d8 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt @@ -8,7 +8,5 @@ add_subdirectory(relations_bench) add_subdirectory(sumcheck_bench) add_subdirectory(translator_bench) add_subdirectory(poseidon2_bench) -add_subdirectory(indexed_tree_bench) -add_subdirectory(append_only_tree_bench) add_subdirectory(ultra_bench) add_subdirectory(circuit_construction_bench) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt deleted file mode 100644 index 7182d4b4be0d..000000000000 --- a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -barretenberg_module(append_only_tree_bench crypto_poseidon2 ext_starknet_crypto_poseidon crypto_pedersen_hash crypto_merkle_tree) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt deleted file mode 100644 index 37ac8e31a80c..000000000000 --- a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -barretenberg_module(indexed_tree_bench crypto_poseidon2 ext_starknet_crypto_poseidon crypto_pedersen_hash crypto_merkle_tree) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt index 55272c0b51b2..f7af171f7984 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt @@ -1,7 +1,6 @@ # merkle tree is agnostic to hash function barretenberg_module( crypto_merkle_tree - lmdblib ) # Need -fPIC because it's linked into the nodejs_module shared library set_target_properties(crypto_merkle_tree_objects PROPERTIES POSITION_INDEPENDENT_CODE ON) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_leaf.hpp similarity index 100% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_leaf.hpp diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp index 420d92f163cf..6f2c70cbd44a 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp @@ -7,64 +7,15 @@ #pragma once #include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/ecc/curves/bn254/fr.hpp" #include "barretenberg/serialize/msgpack.hpp" -#include -#include -#include -#include -#include -#include +#include +// The async tree engine (content-addressed trees + world_state) that produced the full set of +// merkle response/callback types now lives in the wsdb native package; barretenberg keeps only the +// types the AVM's LowLevelMerkleDBInterface returns and the ones its hints serialize. namespace bb::crypto::merkle_tree { -struct TreeMetaResponse { - TreeMeta meta; - - TreeMetaResponse() = default; - ~TreeMetaResponse() = default; - TreeMetaResponse(const TreeMetaResponse& other) = default; - TreeMetaResponse(TreeMetaResponse&& other) noexcept = default; - TreeMetaResponse& operator=(const TreeMetaResponse& other) = default; - TreeMetaResponse& operator=(TreeMetaResponse&& other) noexcept = default; -}; - -struct CheckpointResponse { - uint32_t depth; - - CheckpointResponse() = default; - ~CheckpointResponse() = default; - CheckpointResponse(const CheckpointResponse& other) = default; - CheckpointResponse(CheckpointResponse&& other) noexcept = default; - CheckpointResponse& operator=(const CheckpointResponse& other) = default; - CheckpointResponse& operator=(CheckpointResponse&& other) noexcept = default; -}; - -struct AddDataResponse { - index_t size; - fr root; - - AddDataResponse() = default; - ~AddDataResponse() = default; - AddDataResponse(const AddDataResponse& other) = default; - AddDataResponse(AddDataResponse&& other) noexcept = default; - AddDataResponse& operator=(const AddDataResponse& other) = default; - AddDataResponse& operator=(AddDataResponse&& other) noexcept = default; -}; - -struct GetSiblingPathResponse { - fr_sibling_path path; - - GetSiblingPathResponse() = default; - ~GetSiblingPathResponse() = default; - GetSiblingPathResponse(const GetSiblingPathResponse& other) = default; - GetSiblingPathResponse(GetSiblingPathResponse&& other) noexcept = default; - GetSiblingPathResponse& operator=(const GetSiblingPathResponse& other) = default; - GetSiblingPathResponse& operator=(GetSiblingPathResponse&& other) noexcept = default; -}; template struct LeafUpdateWitnessData { IndexedLeaf leaf; @@ -84,6 +35,7 @@ template struct LeafUpdateWitnessData { LeafUpdateWitnessData& operator=(LeafUpdateWitnessData&& other) noexcept = default; bool operator==(const LeafUpdateWitnessData& other) const = default; + // Serialized as part of the AVM's SequentialInsertHint (see vm2/common/avm_io.hpp). SERIALIZATION_FIELDS(leaf, index, path); }; @@ -91,116 +43,17 @@ template struct BatchInsertionResult { std::vector> low_leaf_witness_data; std::vector> sorted_leaves; fr_sibling_path subtree_path; - - SERIALIZATION_FIELDS(low_leaf_witness_data, sorted_leaves, subtree_path); }; template struct SequentialInsertionResult { std::vector> low_leaf_witness_data; std::vector> insertion_witness_data; - - SERIALIZATION_FIELDS(low_leaf_witness_data, insertion_witness_data); -}; - -template struct AddIndexedDataResponse { - AddDataResponse add_data_result; - fr_sibling_path subtree_path; - std::shared_ptr>> sorted_leaves; - std::shared_ptr>> low_leaf_witness_data; - - AddIndexedDataResponse() = default; - ~AddIndexedDataResponse() = default; - AddIndexedDataResponse(const AddIndexedDataResponse& other) = default; - AddIndexedDataResponse(AddIndexedDataResponse&& other) noexcept = default; - AddIndexedDataResponse& operator=(const AddIndexedDataResponse& other) = default; - AddIndexedDataResponse& operator=(AddIndexedDataResponse&& other) noexcept = default; -}; - -template struct AddIndexedDataSequentiallyResponse { - AddDataResponse add_data_result; - std::shared_ptr>> low_leaf_witness_data; - std::shared_ptr>> insertion_witness_data; - - AddIndexedDataSequentiallyResponse() = default; - ~AddIndexedDataSequentiallyResponse() = default; - AddIndexedDataSequentiallyResponse(const AddIndexedDataSequentiallyResponse& other) = default; - AddIndexedDataSequentiallyResponse(AddIndexedDataSequentiallyResponse&& other) noexcept = default; - AddIndexedDataSequentiallyResponse& operator=(const AddIndexedDataSequentiallyResponse& other) = default; - AddIndexedDataSequentiallyResponse& operator=(AddIndexedDataSequentiallyResponse&& other) noexcept = default; -}; - -struct BlockForIndexResponse { - std::vector> blockNumbers; - - BlockForIndexResponse() = default; - ~BlockForIndexResponse() = default; - BlockForIndexResponse(const BlockForIndexResponse& other) = default; - BlockForIndexResponse(BlockForIndexResponse&& other) noexcept = default; - BlockForIndexResponse& operator=(const BlockForIndexResponse& other) = default; - BlockForIndexResponse& operator=(BlockForIndexResponse&& other) noexcept = default; -}; - -struct FindLeafIndexResponse { - std::vector> leaf_indices; - - FindLeafIndexResponse() = default; - ~FindLeafIndexResponse() = default; - FindLeafIndexResponse(const FindLeafIndexResponse& other) = default; - FindLeafIndexResponse(FindLeafIndexResponse&& other) noexcept = default; - FindLeafIndexResponse& operator=(const FindLeafIndexResponse& other) = default; - FindLeafIndexResponse& operator=(FindLeafIndexResponse&& other) noexcept = default; -}; - -struct SiblingPathAndIndex { - index_t index; - fr_sibling_path path; - - SERIALIZATION_FIELDS(index, path); - - SiblingPathAndIndex() = default; - ~SiblingPathAndIndex() = default; - SiblingPathAndIndex(index_t index, fr_sibling_path path) - : index(index) - , path(path) - {} - SiblingPathAndIndex(const SiblingPathAndIndex& other) = default; - SiblingPathAndIndex(SiblingPathAndIndex&& other) noexcept = default; - SiblingPathAndIndex& operator=(const SiblingPathAndIndex& other) = default; - SiblingPathAndIndex& operator=(SiblingPathAndIndex&& other) noexcept = default; -}; - -struct FindLeafPathResponse { - std::vector> leaf_paths; - - FindLeafPathResponse() = default; - ~FindLeafPathResponse() = default; - FindLeafPathResponse(const FindLeafPathResponse& other) = default; - FindLeafPathResponse(FindLeafPathResponse&& other) noexcept = default; - FindLeafPathResponse& operator=(const FindLeafPathResponse& other) = default; - FindLeafPathResponse& operator=(FindLeafPathResponse&& other) noexcept = default; -}; - -struct GetLeafResponse { - std::optional leaf; - - GetLeafResponse() = default; - ~GetLeafResponse() = default; - GetLeafResponse(const GetLeafResponse& other) = default; - GetLeafResponse(GetLeafResponse&& other) noexcept = default; - GetLeafResponse& operator=(const GetLeafResponse& other) = default; - GetLeafResponse& operator=(GetLeafResponse&& other) noexcept = default; -}; - -template struct GetIndexedLeafResponse { - std::optional> indexed_leaf; }; struct GetLowIndexedLeafResponse { bool is_already_present; index_t index; - SERIALIZATION_FIELDS(is_already_present, index); - GetLowIndexedLeafResponse(bool p, const index_t& i) : is_already_present(p) , index(i) @@ -218,106 +71,4 @@ struct GetLowIndexedLeafResponse { } }; -struct CommitResponse { - TreeMeta meta; - TreeDBStats stats; - - CommitResponse() = default; - ~CommitResponse() = default; - CommitResponse(const CommitResponse& other) = default; - CommitResponse(CommitResponse&& other) noexcept = default; - CommitResponse& operator=(const CommitResponse& other) = default; - CommitResponse& operator=(CommitResponse&& other) noexcept = default; -}; - -struct UnwindResponse { - TreeMeta meta; - TreeDBStats stats; - - UnwindResponse() = default; - ~UnwindResponse() = default; - UnwindResponse(const UnwindResponse& other) = default; - UnwindResponse(UnwindResponse&& other) noexcept = default; - UnwindResponse& operator=(const UnwindResponse& other) = default; - UnwindResponse& operator=(UnwindResponse&& other) noexcept = default; -}; - -struct RemoveHistoricResponse { - TreeMeta meta; - TreeDBStats stats; - - RemoveHistoricResponse() = default; - ~RemoveHistoricResponse() = default; - RemoveHistoricResponse(const RemoveHistoricResponse& other) = default; - RemoveHistoricResponse(RemoveHistoricResponse&& other) noexcept = default; - RemoveHistoricResponse& operator=(const RemoveHistoricResponse& other) = default; - RemoveHistoricResponse& operator=(RemoveHistoricResponse&& other) noexcept = default; -}; - -template struct TypedResponse { - ResponseType inner; - bool success{ true }; - std::string message; - - TypedResponse() = default; - ~TypedResponse() = default; - TypedResponse(const TypedResponse& other) = default; - TypedResponse(TypedResponse&& other) noexcept = default; - TypedResponse& operator=(const TypedResponse& other) = default; - TypedResponse& operator=(TypedResponse&& other) noexcept = default; -}; - -struct Response { - bool success; - std::string message; - - Response(bool s, std::string m) - : success(s) - , message(std::move(m)) - {} - Response() = default; - ~Response() = default; - Response(const Response& other) = default; - Response(Response&& other) noexcept = default; - Response& operator=(const Response& other) = default; - Response& operator=(Response&& other) noexcept = default; -}; - -template -void execute_and_report(const std::function&)>& f, - const std::function&)>& on_completion) -{ - TypedResponse response; - try { - f(response); - } catch (std::exception& e) { - response.success = false; - response.message = e.what(); - // std::cout << "Response " << e.what() << std::endl; - } - try { - on_completion(response); - } catch (std::exception& e) { - std::cerr << "Completion callback threw: " << e.what() << std::endl; - std::abort(); - } -} - -inline void execute_and_report(const std::function& f, const std::function& on_completion) -{ - Response response{ true, "" }; - try { - f(); - } catch (std::exception& e) { - response.success = false; - response.message = e.what(); - // std::cout << "Response " << e.what() << std::endl; - } - try { - on_completion(response); - } catch (std::exception& e) { - std::cerr << "Completion callback threw: " << e.what() << std::endl; - std::abort(); - } -} } // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp index 2af3fb57c206..edf724c4213c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp @@ -7,14 +7,10 @@ #pragma once #include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/lmdblib/types.hpp" -#include "lmdb.h" #include #include namespace bb::crypto::merkle_tree { -using namespace bb::lmdblib; - using index_t = uint64_t; using block_number_t = uint32_t; using LeafIndexKeyType = uint64_t; @@ -65,81 +61,4 @@ const std::string LEAF_PREIMAGES_DB = "leaf preimages"; const std::string LEAF_INDICES_DB = "leaf indices"; const std::string BLOCK_INDICES_DB = "block indices"; -struct TreeDBStats { - uint64_t mapSize; - uint64_t physicalFileSize; - DBStats blocksDBStats; - DBStats nodesDBStats; - DBStats leafPreimagesDBStats; - DBStats leafIndicesDBStats; - DBStats blockIndicesDBStats; - - TreeDBStats() = default; - TreeDBStats(uint64_t mapSize, uint64_t physicalFileSize) - : mapSize(mapSize) - , physicalFileSize(physicalFileSize) - {} - TreeDBStats(uint64_t mapSize, - uint64_t physicalFileSize, - const DBStats& blockStats, - const DBStats& nodesStats, - const DBStats& leafPreimagesDBStats, - const DBStats& leafIndicesStats, - const DBStats& blockIndicesStats) - : mapSize(mapSize) - , physicalFileSize(physicalFileSize) - , blocksDBStats(blockStats) - , nodesDBStats(nodesStats) - , leafPreimagesDBStats(leafPreimagesDBStats) - , leafIndicesDBStats(leafIndicesStats) - , blockIndicesDBStats(blockIndicesStats) - {} - TreeDBStats(const TreeDBStats& other) = default; - TreeDBStats(TreeDBStats&& other) noexcept { *this = std::move(other); } - - ~TreeDBStats() = default; - - SERIALIZATION_FIELDS(mapSize, - physicalFileSize, - blocksDBStats, - nodesDBStats, - leafPreimagesDBStats, - leafIndicesDBStats, - blockIndicesDBStats) - - bool operator==(const TreeDBStats& other) const - { - return mapSize == other.mapSize && physicalFileSize == other.physicalFileSize && - blocksDBStats == other.blocksDBStats && nodesDBStats == other.nodesDBStats && - leafPreimagesDBStats == other.leafPreimagesDBStats && leafIndicesDBStats == other.leafIndicesDBStats && - blockIndicesDBStats == other.blockIndicesDBStats; - } - - TreeDBStats& operator=(TreeDBStats&& other) noexcept - { - if (this != &other) { - mapSize = other.mapSize; - physicalFileSize = other.physicalFileSize; - blocksDBStats = std::move(other.blocksDBStats); - nodesDBStats = std::move(other.nodesDBStats); - leafPreimagesDBStats = std::move(other.leafPreimagesDBStats); - leafIndicesDBStats = std::move(other.leafIndicesDBStats); - blockIndicesDBStats = std::move(other.blockIndicesDBStats); - } - return *this; - } - - TreeDBStats& operator=(const TreeDBStats& other) = default; - - friend std::ostream& operator<<(std::ostream& os, const TreeDBStats& stats) - { - os << "Map Size: " << stats.mapSize << ", Physical File Size: " << stats.physicalFileSize << " Blocks DB " - << stats.blocksDBStats << ", Nodes DB " << stats.nodesDBStats << ", Leaf Pre-images DB " - << stats.leafPreimagesDBStats << ", Leaf Indices DB " << stats.leafIndicesDBStats << ", Block Indices DB " - << stats.blockIndicesDBStats; - return os; - } -}; - -std::ostream& operator<<(std::ostream& os, const TreeDBStats& stats); } // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/poseidon2/poseidon2_capi.cpp b/barretenberg/cpp/src/barretenberg/crypto/poseidon2/poseidon2_capi.cpp new file mode 100644 index 000000000000..56560c9253bf --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/crypto/poseidon2/poseidon2_capi.cpp @@ -0,0 +1,39 @@ +#include "barretenberg/crypto/poseidon2/poseidon2.hpp" +#include "barretenberg/crypto/poseidon2/poseidon2_params.hpp" +#include "barretenberg/ecc/curves/bn254/fr.hpp" +#include +#include +#include + +// Flat C ABI over Poseidon2, for in-process consumers that must hash at high +// frequency but want no dependency on barretenberg's C++ headers, field type, +// stdlib, or build flags (e.g. the standalone world-state DB service). Field +// elements cross as 32 canonical bytes (barretenberg's field serialization); +// the montgomery form stays private to barretenberg. +namespace { +using FF = bb::fr; +using Poseidon2Bn254 = bb::crypto::Poseidon2; +} // namespace + +extern "C" { + +// Hash `n` field elements. `elems` is n*32 canonical bytes; `out` receives 32. +void bb_poseidon2_hash(const uint8_t* elems, size_t n, uint8_t* out) +{ + std::vector inputs; + inputs.reserve(n); + for (size_t i = 0; i < n; ++i) { + inputs.push_back(FF::serialize_from_buffer(elems + (i * 32))); + } + FF::serialize_to_buffer(Poseidon2Bn254::hash(inputs), out); +} + +// Hash { separator, lhs, rhs } — matches Poseidon2HashPolicy::hash_pair_with_separator, +// the domain-separated internal-node hash used by the Aztec merkle trees. +void bb_poseidon2_hash_pair_with_separator(uint64_t separator, const uint8_t* lhs, const uint8_t* rhs, uint8_t* out) +{ + std::vector inputs{ FF(separator), FF::serialize_from_buffer(lhs), FF::serialize_from_buffer(rhs) }; + FF::serialize_to_buffer(Poseidon2Bn254::hash(inputs), out); +} + +} // extern "C" diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/lmdblib/CMakeLists.txt deleted file mode 100644 index efe372413808..000000000000 --- a/barretenberg/cpp/src/barretenberg/lmdblib/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -barretenberg_module(lmdblib lmdb numeric) -set_target_properties(lmdblib_objects PROPERTIES POSITION_INDEPENDENT_CODE ON) diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/fixtures.hpp b/barretenberg/cpp/src/barretenberg/lmdblib/fixtures.hpp deleted file mode 100644 index 2e6cc670dba4..000000000000 --- a/barretenberg/cpp/src/barretenberg/lmdblib/fixtures.hpp +++ /dev/null @@ -1,40 +0,0 @@ -#include "barretenberg/lmdblib/types.hpp" -#include "barretenberg/numeric/random/engine.hpp" -#include - -namespace bb::lmdblib { -const uint32_t NUM_VALUES = 1024; -inline auto& engine = numeric::get_debug_randomness(); -inline auto& random_engine = numeric::get_randomness(); - -inline std::string random_string() -{ - std::stringstream ss; - ss << random_engine.get_random_uint32(); - return ss.str(); -} - -inline std::string random_temp_directory() -{ - std::stringstream ss; - ss << "/tmp/lmdb/" << random_string(); - return ss.str(); -} - -inline std::vector serialise(std::string key) -{ - std::vector data(key.begin(), key.end()); - return data; -} - -inline Key get_key(int64_t keyCount) -{ - return serialise((std::stringstream() << "Key" << keyCount).str()); -} - -inline Value get_value(int64_t keyCount, int64_t valueCount) -{ - return serialise((std::stringstream() << "Key" << keyCount << "Data" << valueCount).str()); -} - -} // namespace bb::lmdblib \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_helpers.cpp b/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_helpers.cpp deleted file mode 100644 index fa4932a7d649..000000000000 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_helpers.cpp +++ /dev/null @@ -1,83 +0,0 @@ -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/numeric/uint256/uint256.hpp" -#include "lmdb.h" -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#define htole64(x) OSSwapHostToLittleInt64(x) -#define le64toh(x) OSSwapLittleToHostInt64(x) -#else -#include -#endif - -namespace bb::lmdblib { -void throw_error(const std::string& errorString, int error) -{ - std::stringstream ss; - ss << errorString << ": " << error << " - " << mdb_strerror(error) << std::endl; - throw std::runtime_error(ss.str()); -} - -std::vector serialise_key(uint8_t key) -{ - return { key }; -} - -void deserialise_key(void* data, uint8_t& key) -{ - uint8_t* p = static_cast(data); - key = *p; -} - -// 64 bit integers are stored in little endian byte order -std::vector serialise_key(uint64_t key) -{ - uint64_t le = htole64(key); - const uint8_t* p = reinterpret_cast(&le); - return std::vector(p, p + sizeof(key)); -} - -void deserialise_key(void* data, uint64_t& key) -{ - uint64_t le = 0; - std::memcpy(&le, data, sizeof(le)); - key = le64toh(le); -} - -std::vector serialise_key(const uint256_t& key) -{ - std::vector buf(32); - std::memcpy(buf.data(), key.data, 32); - return buf; -} - -void deserialise_key(void* data, uint256_t& key) -{ - std::memcpy(key.data, data, 32); -} - -int size_cmp(const MDB_val* a, const MDB_val* b) -{ - if (a->mv_size < b->mv_size) { - return -1; - } - return (a->mv_size > b->mv_size) ? 1 : 0; -} - -std::vector mdb_val_to_vector(const MDB_val& dbVal) -{ - const uint8_t* p = static_cast(dbVal.mv_data); - return std::vector(p, p + dbVal.mv_size); -} - -void copy_to_vector(const MDB_val& dbVal, std::vector& target) -{ - std::vector temp = mdb_val_to_vector(dbVal); - target.swap(temp); -} -} // namespace bb::lmdblib \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_helpers.hpp b/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_helpers.hpp deleted file mode 100644 index 6117fc94a6f0..000000000000 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_helpers.hpp +++ /dev/null @@ -1,61 +0,0 @@ - -#pragma once -#include "barretenberg/numeric/uint256/uint256.hpp" -#include "lmdb.h" -#include -#include -namespace bb::lmdblib { -void throw_error(const std::string& errorString, int error); - -int size_cmp(const MDB_val* a, const MDB_val* b); - -std::vector serialise_key(uint8_t key); -std::vector serialise_key(uint64_t key); -std::vector serialise_key(const uint256_t& key); - -void deserialise_key(void* data, uint8_t& key); -void deserialise_key(void* data, uint64_t& key); -void deserialise_key(void* data, uint256_t& key); - -template int value_cmp(const MDB_val* a, const MDB_val* b) -{ - T lhs; - T rhs; - deserialise_key(a->mv_data, lhs); - deserialise_key(b->mv_data, rhs); - if (lhs < rhs) { - return -1; - } - if (lhs > rhs) { - return 1; - } - return 0; -} - -std::vector mdb_val_to_vector(const MDB_val& dbVal); -void copy_to_vector(const MDB_val& dbVal, std::vector& target); - -template bool call_lmdb_func(int (*f)(TArgs...), TArgs... args) -{ - int error = f(args...); - return error == 0; -} - -template int call_lmdb_func_with_return(int (*f)(TArgs...), TArgs... args) -{ - return f(args...); -} - -template void call_lmdb_func(const std::string& errorString, int (*f)(TArgs...), TArgs... args) -{ - int error = f(args...); - if (error != 0) { - throw_error(errorString, error); - } -} - -template void call_lmdb_func(void (*f)(TArgs...), TArgs... args) -{ - f(args...); -} -} // namespace bb::lmdblib \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/types.hpp b/barretenberg/cpp/src/barretenberg/lmdblib/types.hpp deleted file mode 100644 index 68b48091f2de..000000000000 --- a/barretenberg/cpp/src/barretenberg/lmdblib/types.hpp +++ /dev/null @@ -1,69 +0,0 @@ -#pragma once - -#include "barretenberg/serialize/msgpack.hpp" -#include "lmdb.h" -#include -#include -#include -#include -#include -namespace bb::lmdblib { -using Key = std::vector; -using Value = std::vector; -using KeysVector = std::vector; -using ValuesVector = std::vector; -using KeyValuesPair = std::pair; -using OptionalValues = std::optional; -using OptionalValuesVector = std::vector; -using KeyDupValuesVector = std::vector; -using KeyOptionalValuesPair = std::pair; -using KeyOptionalValuesVector = std::vector; - -struct DBStats { - std::string name; - uint64_t numDataItems; - uint64_t totalUsedSize; - - DBStats() = default; - DBStats(const DBStats& other) = default; - DBStats(DBStats&& other) noexcept { *this = std::move(other); } - ~DBStats() = default; - DBStats(std::string name, MDB_stat& stat) - : name(std::move(name)) - , numDataItems(stat.ms_entries) - , totalUsedSize(stat.ms_psize * (stat.ms_branch_pages + stat.ms_leaf_pages + stat.ms_overflow_pages)) - {} - DBStats(const std::string& name, uint64_t numDataItems, uint64_t totalUsedSize) - : name(name) - , numDataItems(numDataItems) - , totalUsedSize(totalUsedSize) - {} - - SERIALIZATION_FIELDS(name, numDataItems, totalUsedSize) - - bool operator==(const DBStats& other) const - { - return name == other.name && numDataItems == other.numDataItems && totalUsedSize == other.totalUsedSize; - } - - DBStats& operator=(const DBStats& other) = default; - - DBStats& operator=(DBStats&& other) noexcept - { - if (this != &other) { - name = std::move(other.name); - numDataItems = other.numDataItems; - totalUsedSize = other.totalUsedSize; - } - return *this; - } - - friend std::ostream& operator<<(std::ostream& os, const DBStats& stats) - { - os << "DB " << stats.name << ", num items: " << stats.numDataItems - << ", total used size: " << stats.totalUsedSize; - return os; - } -}; - -} // namespace bb::lmdblib \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/nodejs_module/CMakeLists.txt index 736f67b83f2b..77d07c8119eb 100644 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/nodejs_module/CMakeLists.txt @@ -27,7 +27,7 @@ string(REGEX REPLACE "[\r\n\"]" "" NODE_API_HEADERS_DIR ${NODE_API_HEADERS_DIR}) add_library(nodejs_module SHARED ${SOURCE_FILES}) set_target_properties(nodejs_module PROPERTIES PREFIX "" SUFFIX ".node") target_include_directories(nodejs_module PRIVATE ${NODE_API_HEADERS_DIR} ${NODE_ADDON_API_DIR}) -target_link_libraries(nodejs_module PRIVATE ipc ipc_runtime lmdblib) +target_link_libraries(nodejs_module PRIVATE ipc ipc_runtime) # On macOS, Node.js N-API symbols are provided by the runtime, not at link time if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/init_module.cpp b/barretenberg/cpp/src/barretenberg/nodejs_module/init_module.cpp index 1a5a0d0dd396..5dcc6eade9a6 100644 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/init_module.cpp +++ b/barretenberg/cpp/src/barretenberg/nodejs_module/init_module.cpp @@ -1,11 +1,9 @@ -#include "barretenberg/nodejs_module/lmdb_store/lmdb_store_wrapper.hpp" #include "barretenberg/nodejs_module/msgpack_client/msgpack_client_async.hpp" #include "barretenberg/nodejs_module/msgpack_client/msgpack_client_wrapper.hpp" #include "napi.h" Napi::Object Init(Napi::Env env, Napi::Object exports) { - exports.Set(Napi::String::New(env, "LMDBStore"), bb::nodejs::lmdb_store::LMDBStoreWrapper::get_class(env)); exports.Set(Napi::String::New(env, "MsgpackClient"), bb::nodejs::msgpack_client::MsgpackClientWrapper::get_class(env)); exports.Set(Napi::String::New(env, "MsgpackClientAsync"), diff --git a/barretenberg/cpp/src/barretenberg/vm2/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/vm2/CMakeLists.txt index 5fb004640f6c..8f1b2123e950 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/vm2/CMakeLists.txt @@ -53,10 +53,8 @@ if(AVM) # vm2_tests needs dsl for integration testing, but this creates circular dependency: # vm2 provides functions that dsl calls, but tests need dsl functions. # Solution: Link dsl, then vm2 again to resolve the circular reference. - # world_state is a test-only dependency: memory_merkle_db.test.cpp checks MemoryMerkleDB against a - # real WorldState, but vm2 itself no longer depends on world_state. if(NOT WASM AND NOT FUZZING) - target_link_libraries(vm2_tests PRIVATE dsl vm2 world_state) + target_link_libraries(vm2_tests PRIVATE dsl vm2) endif() if(FUZZING) diff --git a/barretenberg/cpp/src/barretenberg/vm2/common/avm_io.cpp b/barretenberg/cpp/src/barretenberg/vm2/common/avm_io.cpp index 0baaa53fbd58..d9494019dafb 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/common/avm_io.cpp +++ b/barretenberg/cpp/src/barretenberg/vm2/common/avm_io.cpp @@ -4,6 +4,7 @@ #include "barretenberg/aztec/aztec_constants.hpp" #include "barretenberg/serialize/msgpack.hpp" +#include "barretenberg/serialize/msgpack_impl.hpp" namespace bb::avm2 { namespace { diff --git a/barretenberg/cpp/src/barretenberg/vm2/common/avm_io.hpp b/barretenberg/cpp/src/barretenberg/vm2/common/avm_io.hpp index ffe7905dcadb..e212a47756da 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/common/avm_io.hpp +++ b/barretenberg/cpp/src/barretenberg/vm2/common/avm_io.hpp @@ -6,7 +6,7 @@ #include "barretenberg/common/streams.hpp" // Derives operator<< from SERIALIZATION_FIELDS. #include "barretenberg/common/utils.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/merkle_tree_id.hpp" // For MerkleTreeId, WorldStateRevision and MSGPACK_ADD_ENUM(MerkleTreeId). #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/serialize/msgpack.hpp" diff --git a/barretenberg/cpp/src/barretenberg/vm2/simulation/gadgets/update_check.test.cpp b/barretenberg/cpp/src/barretenberg/vm2/simulation/gadgets/update_check.test.cpp index 9a8693fd9f2b..088c7b1a56dd 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/simulation/gadgets/update_check.test.cpp +++ b/barretenberg/cpp/src/barretenberg/vm2/simulation/gadgets/update_check.test.cpp @@ -212,8 +212,8 @@ TEST_P(UpdateCheckHashNonzeroTest, WithHash) EXPECT_CALL(merkle_db, get_tree_state()).WillRepeatedly(Return(tree_states)); EXPECT_CALL(merkle_db, as_unconstrained()).WillRepeatedly(ReturnRef(mock_low_level_merkle_db)); - EXPECT_CALL(mock_low_level_merkle_db, get_low_indexed_leaf(world_state::MerkleTreeId::PUBLIC_DATA_TREE, _)) - .WillRepeatedly([&](world_state::MerkleTreeId, const FF& leaf_slot) { + EXPECT_CALL(mock_low_level_merkle_db, get_low_indexed_leaf(crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, _)) + .WillRepeatedly([&](crypto::merkle_tree::MerkleTreeId, const FF& leaf_slot) { for (size_t i = 0; i < update_preimage_slots.size(); ++i) { if (leaf_slot == update_preimage_slots[i]) { return GetLowIndexedLeafResponse(true, static_cast(i)); @@ -297,8 +297,8 @@ TEST(AvmSimulationUpdateCheck, HashMismatch) EXPECT_CALL(mock_low_level_merkle_db, get_tree_roots()).WillRepeatedly(Return(trees)); EXPECT_CALL(merkle_db, as_unconstrained()).WillRepeatedly(ReturnRef(mock_low_level_merkle_db)); - EXPECT_CALL(mock_low_level_merkle_db, get_low_indexed_leaf(world_state::MerkleTreeId::PUBLIC_DATA_TREE, _)) - .WillRepeatedly([&](world_state::MerkleTreeId, const FF& leaf_slot) { + EXPECT_CALL(mock_low_level_merkle_db, get_low_indexed_leaf(crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, _)) + .WillRepeatedly([&](crypto::merkle_tree::MerkleTreeId, const FF& leaf_slot) { return GetLowIndexedLeafResponse(true, static_cast(leaf_slot - delayed_public_mutable_leaf_slot)); }); diff --git a/barretenberg/cpp/src/barretenberg/vm2/simulation/interfaces/db.hpp b/barretenberg/cpp/src/barretenberg/vm2/simulation/interfaces/db.hpp index 284dc69173d7..7fadf284ec19 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/simulation/interfaces/db.hpp +++ b/barretenberg/cpp/src/barretenberg/vm2/simulation/interfaces/db.hpp @@ -4,7 +4,7 @@ #include #include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/merkle_tree_id.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" diff --git a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/db_types.hpp b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/db_types.hpp index 1d5e12091595..4de9e949be8d 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/db_types.hpp +++ b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/db_types.hpp @@ -1,6 +1,6 @@ #pragma once -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/merkle_tree_id.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/vm2/common/aztec_types.hpp" diff --git a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/hinting_dbs.hpp b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/hinting_dbs.hpp index 5e7cb7d79a5d..a51942c16a07 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/hinting_dbs.hpp +++ b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/hinting_dbs.hpp @@ -5,7 +5,7 @@ #include #include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/vm2/common/avm_io.hpp" #include "barretenberg/vm2/common/aztec_types.hpp" diff --git a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/hinting_dbs.test.cpp b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/hinting_dbs.test.cpp index d78e999e8322..ccef47a7dfa1 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/hinting_dbs.test.cpp +++ b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/hinting_dbs.test.cpp @@ -214,8 +214,8 @@ TEST_F(MockedHintingDBsTest, GetLowLeaf) std::vector update_preimage_slots = { 1, 2, 4 }; // get_low_indexed_leaf will call get_tree_roots: EXPECT_CALL(base_merkle_db, get_tree_roots).Times(static_cast(update_preimage_slots.size())); - EXPECT_CALL(base_merkle_db, get_low_indexed_leaf(world_state::MerkleTreeId::PUBLIC_DATA_TREE, ::testing::_)) - .WillRepeatedly([&](world_state::MerkleTreeId, const FF& leaf_slot) { + EXPECT_CALL(base_merkle_db, get_low_indexed_leaf(crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, ::testing::_)) + .WillRepeatedly([&](crypto::merkle_tree::MerkleTreeId, const FF& leaf_slot) { for (size_t i = 0; i < update_preimage_slots.size(); ++i) { if (leaf_slot == update_preimage_slots[i]) { return GetLowIndexedLeafResponse(true, static_cast(i)); @@ -226,30 +226,31 @@ TEST_F(MockedHintingDBsTest, GetLowLeaf) // Call the db: for (const auto& update_preimage_slot : update_preimage_slots) { - hinting_merkle_db.get_low_indexed_leaf(world_state::MerkleTreeId::PUBLIC_DATA_TREE, update_preimage_slot); + hinting_merkle_db.get_low_indexed_leaf(crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, + update_preimage_slot); } ExecutionHints collected_hints; hinting_merkle_db.dump_hints(collected_hints); // Check the collected hints: EXPECT_EQ(collected_hints.get_previous_value_index_hints.size(), update_preimage_slots.size()); - EXPECT_THAT( - collected_hints.get_previous_value_index_hints, - ::testing::ElementsAreArray({ GetPreviousValueIndexHint{ .hint_key = mock_tree_info.public_data_tree, - .tree_id = world_state::MerkleTreeId::PUBLIC_DATA_TREE, - .value = update_preimage_slots[0], - .index = 0, - .already_present = true }, - GetPreviousValueIndexHint{ .hint_key = mock_tree_info.public_data_tree, - .tree_id = world_state::MerkleTreeId::PUBLIC_DATA_TREE, - .value = update_preimage_slots[1], - .index = 1, - .already_present = true }, - GetPreviousValueIndexHint{ .hint_key = mock_tree_info.public_data_tree, - .tree_id = world_state::MerkleTreeId::PUBLIC_DATA_TREE, - .value = update_preimage_slots[2], - .index = 2, - .already_present = true } })); + EXPECT_THAT(collected_hints.get_previous_value_index_hints, + ::testing::ElementsAreArray( + { GetPreviousValueIndexHint{ .hint_key = mock_tree_info.public_data_tree, + .tree_id = crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, + .value = update_preimage_slots[0], + .index = 0, + .already_present = true }, + GetPreviousValueIndexHint{ .hint_key = mock_tree_info.public_data_tree, + .tree_id = crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, + .value = update_preimage_slots[1], + .index = 1, + .already_present = true }, + GetPreviousValueIndexHint{ .hint_key = mock_tree_info.public_data_tree, + .tree_id = crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, + .value = update_preimage_slots[2], + .index = 2, + .already_present = true } })); } TEST_F(MockedHintingDBsTest, GetLeafValue) @@ -258,8 +259,8 @@ TEST_F(MockedHintingDBsTest, GetLeafValue) std::vector note_hash_leaf_values = { 11, 22, 44, 88 }; // get_leaf_value will call get_tree_roots: EXPECT_CALL(base_merkle_db, get_tree_roots).Times(static_cast(note_hash_leaf_values.size())); - EXPECT_CALL(base_merkle_db, get_leaf_value(world_state::MerkleTreeId::NOTE_HASH_TREE, ::testing::_)) - .WillRepeatedly([&](world_state::MerkleTreeId, index_t index) { + EXPECT_CALL(base_merkle_db, get_leaf_value(crypto::merkle_tree::MerkleTreeId::NOTE_HASH_TREE, ::testing::_)) + .WillRepeatedly([&](crypto::merkle_tree::MerkleTreeId, index_t index) { if (index < note_hash_leaf_values.size()) { return note_hash_leaf_values[index]; } @@ -268,7 +269,7 @@ TEST_F(MockedHintingDBsTest, GetLeafValue) // Call the db: for (index_t i = 0; i < note_hash_leaf_values.size(); i++) { - hinting_merkle_db.get_leaf_value(world_state::MerkleTreeId::NOTE_HASH_TREE, i); + hinting_merkle_db.get_leaf_value(crypto::merkle_tree::MerkleTreeId::NOTE_HASH_TREE, i); } ExecutionHints collected_hints; hinting_merkle_db.dump_hints(collected_hints); @@ -278,19 +279,19 @@ TEST_F(MockedHintingDBsTest, GetLeafValue) EXPECT_THAT(collected_hints.get_leaf_value_hints, ::testing::ElementsAreArray({ GetLeafValueHint{ .hint_key = mock_tree_info.note_hash_tree, - .tree_id = world_state::MerkleTreeId::NOTE_HASH_TREE, + .tree_id = crypto::merkle_tree::MerkleTreeId::NOTE_HASH_TREE, .index = 0, .value = note_hash_leaf_values[0] }, GetLeafValueHint{ .hint_key = mock_tree_info.note_hash_tree, - .tree_id = world_state::MerkleTreeId::NOTE_HASH_TREE, + .tree_id = crypto::merkle_tree::MerkleTreeId::NOTE_HASH_TREE, .index = 1, .value = note_hash_leaf_values[1] }, GetLeafValueHint{ .hint_key = mock_tree_info.note_hash_tree, - .tree_id = world_state::MerkleTreeId::NOTE_HASH_TREE, + .tree_id = crypto::merkle_tree::MerkleTreeId::NOTE_HASH_TREE, .index = 2, .value = note_hash_leaf_values[2] }, GetLeafValueHint{ .hint_key = mock_tree_info.note_hash_tree, - .tree_id = world_state::MerkleTreeId::NOTE_HASH_TREE, + .tree_id = crypto::merkle_tree::MerkleTreeId::NOTE_HASH_TREE, .index = 3, .value = note_hash_leaf_values[3] }, })); @@ -404,7 +405,7 @@ TEST_F(MockedHintingDBsTest, InsertIndexedLeavesPublicDataTree) EXPECT_THAT(collected_hints.sequential_insert_hints_public_data_tree, ::testing::ElementsAre(SequentialInsertHint{ .hint_key = state_before, - .tree_id = world_state::MerkleTreeId::PUBLIC_DATA_TREE, + .tree_id = crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, .leaf = public_leaf_value, .low_leaves_witness_data = mock_low_witness_data, .insertion_witness_data = { { public_leaf_value, 1, 6 }, 1, mock_path }, @@ -444,7 +445,7 @@ TEST_F(MockedHintingDBsTest, InsertIndexedLeavesNullifierTree) EXPECT_THAT(collected_hints.sequential_insert_hints_nullifier_tree, ::testing::ElementsAre(SequentialInsertHint{ .hint_key = state_before, - .tree_id = world_state::MerkleTreeId::NULLIFIER_TREE, + .tree_id = crypto::merkle_tree::MerkleTreeId::NULLIFIER_TREE, .leaf = nullifier, .low_leaves_witness_data = mock_low_witness_data, .insertion_witness_data = { { nullifier, 1, 6 }, 1, mock_path }, @@ -465,10 +466,11 @@ TEST_F(MockedHintingDBsTest, AppendLeaves) EXPECT_CALL(base_merkle_db, get_tree_roots) .WillOnce(::testing::Return(mock_tree_info)) .WillOnce(::testing::Return(expected_tree_info_after)); - EXPECT_CALL(base_merkle_db, append_leaves(world_state::MerkleTreeId::NOTE_HASH_TREE, ::testing::_)).Times(1); + EXPECT_CALL(base_merkle_db, append_leaves(crypto::merkle_tree::MerkleTreeId::NOTE_HASH_TREE, ::testing::_)) + .Times(1); // Call the db: - hinting_merkle_db.append_leaves(world_state::MerkleTreeId::NOTE_HASH_TREE, note_hash_leaf_values); + hinting_merkle_db.append_leaves(crypto::merkle_tree::MerkleTreeId::NOTE_HASH_TREE, note_hash_leaf_values); ExecutionHints collected_hints; hinting_merkle_db.dump_hints(collected_hints); @@ -477,7 +479,7 @@ TEST_F(MockedHintingDBsTest, AppendLeaves) EXPECT_THAT(collected_hints.append_leaves_hints, ::testing::ElementsAre(AppendLeavesHint{ .hint_key = initial_state, .state_after = expected_end_state, - .tree_id = world_state::MerkleTreeId::NOTE_HASH_TREE, + .tree_id = crypto::merkle_tree::MerkleTreeId::NOTE_HASH_TREE, .leaves = note_hash_leaf_values })); } diff --git a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/memory_merkle_db.hpp b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/memory_merkle_db.hpp index d1a930692ee2..07d2942cd600 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/memory_merkle_db.hpp +++ b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/memory_merkle_db.hpp @@ -8,7 +8,7 @@ #include "barretenberg/aztec/aztec_hash_policy.hpp" #include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/vm2/common/aztec_types.hpp" #include "barretenberg/vm2/common/field.hpp" diff --git a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/raw_data_dbs.cpp b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/raw_data_dbs.cpp index 96bbeaaa3de1..b2b84b4d1922 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/raw_data_dbs.cpp +++ b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/raw_data_dbs.cpp @@ -9,7 +9,7 @@ #include "barretenberg/aztec/aztec_constants.hpp" #include "barretenberg/common/bb_bench.hpp" #include "barretenberg/common/log.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/vm2/common/avm_io.hpp" #include "barretenberg/vm2/simulation/interfaces/db.hpp" #include "barretenberg/vm2/simulation/lib/contract_crypto.hpp" diff --git a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/raw_data_dbs.hpp b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/raw_data_dbs.hpp index 96ef3a15a1c2..3134bb78dff1 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/raw_data_dbs.hpp +++ b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/raw_data_dbs.hpp @@ -5,7 +5,7 @@ #include #include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/vm2/common/avm_io.hpp" #include "barretenberg/vm2/common/aztec_types.hpp" diff --git a/barretenberg/cpp/src/barretenberg/vm2/testing/public_tx_simulation_tester.cpp b/barretenberg/cpp/src/barretenberg/vm2/testing/public_tx_simulation_tester.cpp index 77516df9333a..5dc9debb29d5 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/testing/public_tx_simulation_tester.cpp +++ b/barretenberg/cpp/src/barretenberg/vm2/testing/public_tx_simulation_tester.cpp @@ -1,7 +1,7 @@ #include "barretenberg/vm2/testing/public_tx_simulation_tester.hpp" #include "barretenberg/aztec/aztec_constants.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/crypto/poseidon2/poseidon2.hpp" #include "barretenberg/vm2/simulation/lib/contract_crypto.hpp" #include "barretenberg/vm2/simulation/lib/merkle.hpp" diff --git a/barretenberg/cpp/src/barretenberg/vm2/tracegen/update_check_trace.test.cpp b/barretenberg/cpp/src/barretenberg/vm2/tracegen/update_check_trace.test.cpp index 5d0e96d62610..7a0d7226df00 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/tracegen/update_check_trace.test.cpp +++ b/barretenberg/cpp/src/barretenberg/vm2/tracegen/update_check_trace.test.cpp @@ -135,13 +135,13 @@ TEST(UpdateCheckTracegenTest, HashZeroInteractions) uint32_t leaf_index = 27; EXPECT_CALL(mock_low_level_merkle_db, get_tree_roots()).WillRepeatedly(Return(trees)); - EXPECT_CALL(mock_low_level_merkle_db, get_sibling_path(world_state::MerkleTreeId::PUBLIC_DATA_TREE, _)) + EXPECT_CALL(mock_low_level_merkle_db, get_sibling_path(crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, _)) .WillOnce(Return(fr_sibling_path{ 0 })); EXPECT_CALL(mock_low_level_merkle_db, get_leaf_preimage_public_data_tree(_)) .WillOnce(Return(PublicDataTreeLeafPreimage(PublicDataLeafValue(1, 0), 0, 0))); - EXPECT_CALL( - mock_low_level_merkle_db, - get_low_indexed_leaf(world_state::MerkleTreeId::PUBLIC_DATA_TREE, delayed_public_mutable_hash_leaf_slot)) + EXPECT_CALL(mock_low_level_merkle_db, + get_low_indexed_leaf(crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, + delayed_public_mutable_hash_leaf_slot)) .WillOnce(Return(GetLowIndexedLeafResponse(false, leaf_index))); EXPECT_CALL(mock_field_gt, ff_gt(_, _)).WillRepeatedly(Return(true)); @@ -239,7 +239,7 @@ TEST(UpdateCheckTracegenTest, HashNonzeroInteractions) } EXPECT_CALL(mock_low_level_merkle_db, get_tree_roots()).WillRepeatedly(Return(trees)); - EXPECT_CALL(mock_low_level_merkle_db, get_sibling_path(world_state::MerkleTreeId::PUBLIC_DATA_TREE, _)) + EXPECT_CALL(mock_low_level_merkle_db, get_sibling_path(crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, _)) .WillOnce(Return(fr_sibling_path{ 0 })); EXPECT_CALL(mock_low_level_merkle_db, get_leaf_preimage_public_data_tree(_)) .WillRepeatedly([&](const uint64_t& index) { @@ -247,8 +247,8 @@ TEST(UpdateCheckTracegenTest, HashNonzeroInteractions) PublicDataLeafValue(update_leaf_slots[index], update_leaf_values[index]), 0, 0); }); - EXPECT_CALL(mock_low_level_merkle_db, get_low_indexed_leaf(world_state::MerkleTreeId::PUBLIC_DATA_TREE, _)) - .WillRepeatedly([&](world_state::MerkleTreeId, const FF& leaf_slot) { + EXPECT_CALL(mock_low_level_merkle_db, get_low_indexed_leaf(crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, _)) + .WillRepeatedly([&](crypto::merkle_tree::MerkleTreeId, const FF& leaf_slot) { for (size_t i = 0; i < update_leaf_slots.size(); ++i) { if (leaf_slot == update_leaf_slots[i]) { return GetLowIndexedLeafResponse(true, static_cast(i)); diff --git a/barretenberg/cpp/src/barretenberg/vm2_wsdb/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/vm2_wsdb/CMakeLists.txt index ee67a00907e3..5c639a33791a 100644 --- a/barretenberg/cpp/src/barretenberg/vm2_wsdb/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/vm2_wsdb/CMakeLists.txt @@ -1,16 +1,61 @@ -# VM2 adapter for native AVM simulation against aztec-wsdb. -if(TARGET vm2_sim AND TARGET wsdb_ipc_client) +# VM2 binding to the wsdb service: the generated IPC client plus the AVM adapter +# (WsdbIpcMerkleDB) that drives native AVM simulation against aztec-wsdb. +# +# The wsdb wire contract (wsdb_schema.jsonc) and the server live in the +# top-level native-packages/wsdb package, which owns the service. bb only +# needs the generated CLIENT, so it codegens from that external schema path. +# This is a deliberate (and currently the only) build-order inversion: +# bb reads native-packages/wsdb/wsdb_schema.jsonc at configure/build time. +if(TARGET vm2_sim) + set(WSDB_SCHEMA ${CMAKE_SOURCE_DIR}/../../native-packages/wsdb/wsdb_schema.jsonc) + set(WSDB_GEN_DIR ${CMAKE_CURRENT_SOURCE_DIR}/generated) + set(WSDB_GEN_OUTPUTS + ${WSDB_GEN_DIR}/wsdb_ipc_client.cpp + ${WSDB_GEN_DIR}/wsdb_ipc_client.hpp + ${WSDB_GEN_DIR}/wsdb_types.hpp + ${WSDB_GEN_DIR}/ipc_codegen/msgpack_adaptor.hpp + ${WSDB_GEN_DIR}/ipc_codegen/named_union.hpp + ${WSDB_GEN_DIR}/ipc_codegen/throw.hpp + ) + set(IPC_CODEGEN_DIR ${CMAKE_SOURCE_DIR}/../../ipc-codegen) + file(GLOB_RECURSE IPC_CODEGEN_SRC + ${IPC_CODEGEN_DIR}/src/*.ts + ${IPC_CODEGEN_DIR}/templates/cpp/*.hpp + ) + add_custom_command( + OUTPUT ${WSDB_GEN_OUTPUTS} + COMMAND node --experimental-strip-types --experimental-transform-types --no-warnings + ${IPC_CODEGEN_DIR}/src/generate.ts + --schema ${WSDB_SCHEMA} + --lang cpp + --out ${WSDB_GEN_DIR} + --client + --cpp-namespace bb::wsdb + --cpp-include-dir barretenberg/vm2_wsdb/generated + DEPENDS ${WSDB_SCHEMA} ${IPC_CODEGEN_SRC} + COMMENT "Generating WSDB IPC client from native-packages/wsdb/wsdb_schema.jsonc" + VERBATIM + ) + add_custom_target(wsdb_ipc_generated DEPENDS ${WSDB_GEN_OUTPUTS}) + add_library( wsdb_ipc_merkle_db STATIC wsdb_ipc_merkle_db.cpp + ${WSDB_GEN_DIR}/wsdb_ipc_client.cpp + ) + add_dependencies(wsdb_ipc_merkle_db wsdb_ipc_generated) + target_include_directories( + wsdb_ipc_merkle_db + PUBLIC + ${WSDB_GEN_DIR} ) target_link_libraries( wsdb_ipc_merkle_db PUBLIC barretenberg vm2_sim - wsdb_ipc_client + ipc_runtime ) set_target_properties(wsdb_ipc_merkle_db PROPERTIES POSITION_INDEPENDENT_CODE ON) endif() diff --git a/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_client.hpp b/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_client.hpp new file mode 100644 index 000000000000..38995831838e --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_client.hpp @@ -0,0 +1,3 @@ +#pragma once + +#include "barretenberg/vm2_wsdb/generated/wsdb_ipc_client.hpp" diff --git a/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.cpp b/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.cpp index 912a840ca71e..fb38e20a10b2 100644 --- a/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.cpp +++ b/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.cpp @@ -1,7 +1,7 @@ #include "barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.hpp" #include "barretenberg/aztec/aztec_constants.hpp" #include "barretenberg/common/log.hpp" -#include "barretenberg/wsdb/wsdb_wire_convert.hpp" +#include "barretenberg/vm2_wsdb/wsdb_wire_convert_client.hpp" #include @@ -9,7 +9,6 @@ namespace bb::avm2::simulation { // Wire <-> domain conversion helpers are shared with the server handlers // (see wsdb_handlers.cpp) so both sides use the same encoding boundary. -using bb::wsdb::Fr; using bb::wsdb::fr_from_wire; using bb::wsdb::fr_to_wire; using bb::wsdb::fr_vec_from_wire; @@ -21,12 +20,13 @@ using bb::wsdb::revision_to_wire; using bb::wsdb::sequential_nullifier_from_wire; using bb::wsdb::sequential_public_data_from_wire; using bb::wsdb::tree_id_to_wire; +using bb::wsdb::wire::Fr; // --------------------------------------------------------------------------- // Constructor // --------------------------------------------------------------------------- -WsdbIpcMerkleDB::WsdbIpcMerkleDB(wsdb::WsdbIpcClient& client, world_state::WorldStateRevision revision) +WsdbIpcMerkleDB::WsdbIpcMerkleDB(wsdb::WsdbIpcClient& client, crypto::merkle_tree::WorldStateRevision revision) : client_(client) , revision_(revision) {} diff --git a/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.hpp b/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.hpp index ff2056b8b6ab..c5b8fcb91471 100644 --- a/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.hpp +++ b/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.hpp @@ -7,9 +7,9 @@ * each LowLevelMerkleDBInterface call into the corresponding WSDB IPC command. */ +#include "barretenberg/crypto/merkle_tree/merkle_tree_id.hpp" #include "barretenberg/vm2/simulation/interfaces/db.hpp" -#include "barretenberg/world_state/types.hpp" -#include "barretenberg/wsdb/wsdb_ipc_client.hpp" +#include "barretenberg/vm2_wsdb/wsdb_ipc_client.hpp" #include #include @@ -23,7 +23,7 @@ class WsdbIpcMerkleDB final : public avm2::simulation::LowLevelMerkleDBInterface * @param client Reference to a connected WsdbIpcClient. * @param revision The world state revision (includes forkId) to use for queries. */ - WsdbIpcMerkleDB(wsdb::WsdbIpcClient& client, world_state::WorldStateRevision revision); + WsdbIpcMerkleDB(wsdb::WsdbIpcClient& client, crypto::merkle_tree::WorldStateRevision revision); avm2::TreeSnapshots get_tree_roots() const override; @@ -58,7 +58,7 @@ class WsdbIpcMerkleDB final : public avm2::simulation::LowLevelMerkleDBInterface void invalidate_tree_roots_cache(); wsdb::WsdbIpcClient& client_; - world_state::WorldStateRevision revision_; + crypto::merkle_tree::WorldStateRevision revision_; std::stack checkpoint_stack_{ { 0 } }; /** Cached tree roots — avoids 5 IPC round trips per get_tree_roots() call. */ mutable std::optional cached_tree_roots_; diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_wire_convert.hpp b/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_wire_convert_client.hpp similarity index 54% rename from barretenberg/cpp/src/barretenberg/wsdb/wsdb_wire_convert.hpp rename to barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_wire_convert_client.hpp index 6de14909e38e..97ba8d4e90f4 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_wire_convert.hpp +++ b/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_wire_convert_client.hpp @@ -1,82 +1,87 @@ #pragma once /** * @file wsdb_wire_convert.hpp - * @brief Wire <-> domain conversion helpers for the aztec-wsdb service. + * @brief Client-side wire <-> domain conversion helpers for the aztec-wsdb service. + * + * These converters depend only on the generic merkle-tree vocabulary + * (crypto/merkle_tree) plus the generated wsdb wire types, so they can be + * compiled into bb's wsdb client (consumed by the AVM simulator) without + * pulling in world_state or the persistent merkle storage. The server-side + * converters that touch world_state aggregates (state references, DB stats, + * tree/world-state meta) live alongside the server in native-packages/wsdb. */ -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/merkle_tree_id.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/lmdblib/types.hpp" -#include "barretenberg/world_state/types.hpp" -#include "barretenberg/world_state/world_state.hpp" -#include "barretenberg/wsdb/generated/wsdb_types.hpp" +#include "barretenberg/vm2_wsdb/generated/wsdb_types.hpp" namespace bb::wsdb { -inline Fr fr_to_wire(const bb::fr& d) +inline wire::Fr fr_to_wire(const bb::fr& d) { - Fr r{}; + wire::Fr r{}; bb::fr::serialize_to_buffer(d, r.data()); return r; } -inline bb::fr fr_from_wire(const Fr& w) +inline bb::fr fr_from_wire(const wire::Fr& w) { return bb::fr::serialize_from_buffer(w.data()); } -inline BlockHeaderHash block_header_hash_to_wire(const bb::fr& d) +inline wire::BlockHeaderHash block_header_hash_to_wire(const bb::fr& d) { - BlockHeaderHash r{}; + wire::BlockHeaderHash r{}; bb::fr::serialize_to_buffer(d, r.data()); return r; } -inline bb::fr block_header_hash_from_wire(const BlockHeaderHash& w) +inline bb::fr block_header_hash_from_wire(const wire::BlockHeaderHash& w) { return bb::fr::serialize_from_buffer(w.data()); } -inline PublicDataSlot public_data_slot_to_wire(const bb::fr& d) +inline wire::PublicDataSlot public_data_slot_to_wire(const bb::fr& d) { - PublicDataSlot r{}; + wire::PublicDataSlot r{}; bb::fr::serialize_to_buffer(d, r.data()); return r; } -inline bb::fr public_data_slot_from_wire(const PublicDataSlot& w) +inline bb::fr public_data_slot_from_wire(const wire::PublicDataSlot& w) { return bb::fr::serialize_from_buffer(w.data()); } -inline PublicDataValue public_data_value_to_wire(const bb::fr& d) +inline wire::PublicDataValue public_data_value_to_wire(const bb::fr& d) { - PublicDataValue r{}; + wire::PublicDataValue r{}; bb::fr::serialize_to_buffer(d, r.data()); return r; } -inline bb::fr public_data_value_from_wire(const PublicDataValue& w) +inline bb::fr public_data_value_from_wire(const wire::PublicDataValue& w) { return bb::fr::serialize_from_buffer(w.data()); } -inline Nullifier nullifier_to_wire(const bb::fr& d) +inline wire::Nullifier nullifier_to_wire(const bb::fr& d) { - Nullifier r{}; + wire::Nullifier r{}; bb::fr::serialize_to_buffer(d, r.data()); return r; } -inline bb::fr nullifier_from_wire(const Nullifier& w) +inline bb::fr nullifier_from_wire(const wire::Nullifier& w) { return bb::fr::serialize_from_buffer(w.data()); } -inline std::vector fr_vec_to_wire(const std::vector& d) +inline std::vector fr_vec_to_wire(const std::vector& d) { - std::vector r; + std::vector r; r.reserve(d.size()); for (const auto& x : d) { r.push_back(fr_to_wire(x)); @@ -84,7 +89,7 @@ inline std::vector fr_vec_to_wire(const std::vector& d) return r; } -inline std::vector fr_vec_from_wire(const std::vector& w) +inline std::vector fr_vec_from_wire(const std::vector& w) { std::vector r; r.reserve(w.size()); @@ -94,7 +99,7 @@ inline std::vector fr_vec_from_wire(const std::vector& w) return r; } -inline wire::WorldStateRevision revision_to_wire(const world_state::WorldStateRevision& d) +inline wire::WorldStateRevision revision_to_wire(const crypto::merkle_tree::WorldStateRevision& d) { return wire::WorldStateRevision{ .forkId = d.forkId, @@ -103,23 +108,23 @@ inline wire::WorldStateRevision revision_to_wire(const world_state::WorldStateRe }; } -inline world_state::WorldStateRevision revision_from_wire(const wire::WorldStateRevision& w) +inline crypto::merkle_tree::WorldStateRevision revision_from_wire(const wire::WorldStateRevision& w) { - return world_state::WorldStateRevision{ + return crypto::merkle_tree::WorldStateRevision{ .forkId = w.forkId, .blockNumber = w.blockNumber, .includeUncommitted = w.includeUncommitted, }; } -inline MerkleTreeId tree_id_to_wire(world_state::MerkleTreeId d) +inline wire::MerkleTreeId tree_id_to_wire(crypto::merkle_tree::MerkleTreeId d) { - return static_cast(d); + return static_cast(d); } -inline world_state::MerkleTreeId tree_id_from_wire(MerkleTreeId w) +inline crypto::merkle_tree::MerkleTreeId tree_id_from_wire(wire::MerkleTreeId w) { - return static_cast(w); + return static_cast(w); } inline wire::PublicDataLeafValue public_data_leaf_to_wire(const crypto::merkle_tree::PublicDataLeafValue& d) @@ -257,7 +262,7 @@ inline std::vector vec_from_wire(const std::vector& w, Fn fn) } inline wire::BatchInsertionResultPublicData batch_public_data_to_wire( - const world_state::BatchInsertionResult& d) + const crypto::merkle_tree::BatchInsertionResult& d) { std::vector sorted; sorted.reserve(d.sorted_leaves.size()); @@ -270,10 +275,10 @@ inline wire::BatchInsertionResultPublicData batch_public_data_to_wire( .subtreePath = fr_vec_to_wire(d.subtree_path) }; } -inline world_state::BatchInsertionResult batch_public_data_from_wire( +inline crypto::merkle_tree::BatchInsertionResult batch_public_data_from_wire( const wire::BatchInsertionResultPublicData& w) { - world_state::BatchInsertionResult r; + crypto::merkle_tree::BatchInsertionResult r; r.low_leaf_witness_data = vec_from_wire>( w.lowLeafWitnessData, public_data_witness_from_wire); @@ -286,7 +291,7 @@ inline world_state::BatchInsertionResult& d) + const crypto::merkle_tree::BatchInsertionResult& d) { std::vector sorted; sorted.reserve(d.sorted_leaves.size()); @@ -299,10 +304,10 @@ inline wire::BatchInsertionResultNullifier batch_nullifier_to_wire( .subtreePath = fr_vec_to_wire(d.subtree_path) }; } -inline world_state::BatchInsertionResult batch_nullifier_from_wire( +inline crypto::merkle_tree::BatchInsertionResult batch_nullifier_from_wire( const wire::BatchInsertionResultNullifier& w) { - world_state::BatchInsertionResult r; + crypto::merkle_tree::BatchInsertionResult r; r.low_leaf_witness_data = vec_from_wire>( w.lowLeafWitnessData, nullifier_witness_from_wire); @@ -315,7 +320,7 @@ inline world_state::BatchInsertionResult& d) + const crypto::merkle_tree::SequentialInsertionResult& d) { return { .lowLeafWitnessData = vec_to_wire(d.low_leaf_witness_data, public_data_witness_to_wire), @@ -323,7 +328,7 @@ inline wire::SequentialInsertionResultPublicData sequential_public_data_to_wire( public_data_witness_to_wire) }; } -inline world_state::SequentialInsertionResult +inline crypto::merkle_tree::SequentialInsertionResult sequential_public_data_from_wire(const wire::SequentialInsertionResultPublicData& w) { return { .low_leaf_witness_data = @@ -335,7 +340,7 @@ sequential_public_data_from_wire(const wire::SequentialInsertionResultPublicData } inline wire::SequentialInsertionResultNullifier sequential_nullifier_to_wire( - const world_state::SequentialInsertionResult& d) + const crypto::merkle_tree::SequentialInsertionResult& d) { return { .lowLeafWitnessData = vec_to_wire(d.low_leaf_witness_data, nullifier_witness_to_wire), @@ -343,8 +348,8 @@ inline wire::SequentialInsertionResultNullifier sequential_nullifier_to_wire( nullifier_witness_to_wire) }; } -inline world_state::SequentialInsertionResult sequential_nullifier_from_wire( - const wire::SequentialInsertionResultNullifier& w) +inline crypto::merkle_tree::SequentialInsertionResult +sequential_nullifier_from_wire(const wire::SequentialInsertionResultNullifier& w) { return { .low_leaf_witness_data = vec_from_wire>( @@ -354,151 +359,4 @@ inline world_state::SequentialInsertionResult state_reference_to_wire(const world_state::StateReference& d) -{ - std::vector r; - r.reserve(d.size()); - for (const auto& [tree_id, tree_ref] : d) { - r.push_back( - { .treeId = tree_id_to_wire(tree_id), .root = fr_to_wire(tree_ref.first), .size = tree_ref.second }); - } - return r; -} - -inline world_state::StateReference state_reference_from_wire(const std::vector& w) -{ - world_state::StateReference r; - r.reserve(w.size()); - for (const auto& entry : w) { - r.emplace(tree_id_from_wire(entry.treeId), - world_state::TreeStateReference{ fr_from_wire(entry.root), entry.size }); - } - return r; -} - -inline wire::DBStats db_stats_to_wire(const bb::lmdblib::DBStats& d) -{ - return { .name = d.name, .numDataItems = d.numDataItems, .totalUsedSize = d.totalUsedSize }; -} - -inline bb::lmdblib::DBStats db_stats_from_wire(const wire::DBStats& w) -{ - return bb::lmdblib::DBStats(w.name, w.numDataItems, w.totalUsedSize); -} - -inline wire::TreeDBStats tree_db_stats_to_wire(const bb::crypto::merkle_tree::TreeDBStats& d) -{ - return { .mapSize = d.mapSize, - .physicalFileSize = d.physicalFileSize, - .blocksDBStats = db_stats_to_wire(d.blocksDBStats), - .nodesDBStats = db_stats_to_wire(d.nodesDBStats), - .leafPreimagesDBStats = db_stats_to_wire(d.leafPreimagesDBStats), - .leafIndicesDBStats = db_stats_to_wire(d.leafIndicesDBStats), - .blockIndicesDBStats = db_stats_to_wire(d.blockIndicesDBStats) }; -} - -inline bb::crypto::merkle_tree::TreeDBStats tree_db_stats_from_wire(const wire::TreeDBStats& w) -{ - return { w.mapSize, - w.physicalFileSize, - db_stats_from_wire(w.blocksDBStats), - db_stats_from_wire(w.nodesDBStats), - db_stats_from_wire(w.leafPreimagesDBStats), - db_stats_from_wire(w.leafIndicesDBStats), - db_stats_from_wire(w.blockIndicesDBStats) }; -} - -inline wire::TreeMeta tree_meta_to_wire(const bb::crypto::merkle_tree::TreeMeta& d) -{ - return { .name = d.name, - .depth = d.depth, - .size = d.size, - .committedSize = d.committedSize, - .root = fr_to_wire(d.root), - .initialSize = d.initialSize, - .initialRoot = fr_to_wire(d.initialRoot), - .oldestHistoricBlock = d.oldestHistoricBlock, - .unfinalizedBlockHeight = d.unfinalizedBlockHeight, - .finalizedBlockHeight = d.finalizedBlockHeight }; -} - -inline bb::crypto::merkle_tree::TreeMeta tree_meta_from_wire(const wire::TreeMeta& w) -{ - return { w.name, - w.depth, - w.size, - w.committedSize, - fr_from_wire(w.root), - w.initialSize, - fr_from_wire(w.initialRoot), - w.oldestHistoricBlock, - w.unfinalizedBlockHeight, - w.finalizedBlockHeight }; -} - -inline wire::WorldStateDBStats world_state_db_stats_to_wire(const bb::world_state::WorldStateDBStats& d) -{ - return { .noteHashTreeStats = tree_db_stats_to_wire(d.noteHashTreeStats), - .messageTreeStats = tree_db_stats_to_wire(d.messageTreeStats), - .archiveTreeStats = tree_db_stats_to_wire(d.archiveTreeStats), - .publicDataTreeStats = tree_db_stats_to_wire(d.publicDataTreeStats), - .nullifierTreeStats = tree_db_stats_to_wire(d.nullifierTreeStats) }; -} - -inline bb::world_state::WorldStateDBStats world_state_db_stats_from_wire(const wire::WorldStateDBStats& w) -{ - return { tree_db_stats_from_wire(w.noteHashTreeStats), - tree_db_stats_from_wire(w.messageTreeStats), - tree_db_stats_from_wire(w.archiveTreeStats), - tree_db_stats_from_wire(w.publicDataTreeStats), - tree_db_stats_from_wire(w.nullifierTreeStats) }; -} - -inline wire::WorldStateMeta world_state_meta_to_wire(const bb::world_state::WorldStateMeta& d) -{ - return { .noteHashTreeMeta = tree_meta_to_wire(d.noteHashTreeMeta), - .messageTreeMeta = tree_meta_to_wire(d.messageTreeMeta), - .archiveTreeMeta = tree_meta_to_wire(d.archiveTreeMeta), - .publicDataTreeMeta = tree_meta_to_wire(d.publicDataTreeMeta), - .nullifierTreeMeta = tree_meta_to_wire(d.nullifierTreeMeta) }; -} - -inline bb::world_state::WorldStateMeta world_state_meta_from_wire(const wire::WorldStateMeta& w) -{ - return { tree_meta_from_wire(w.noteHashTreeMeta), - tree_meta_from_wire(w.messageTreeMeta), - tree_meta_from_wire(w.archiveTreeMeta), - tree_meta_from_wire(w.publicDataTreeMeta), - tree_meta_from_wire(w.nullifierTreeMeta) }; -} - -inline wire::WorldStateStatusSummary world_state_status_summary_to_wire( - const bb::world_state::WorldStateStatusSummary& d) -{ - return { .unfinalizedBlockNumber = d.unfinalizedBlockNumber, - .finalizedBlockNumber = d.finalizedBlockNumber, - .oldestHistoricalBlock = d.oldestHistoricalBlock, - .treesAreSynched = d.treesAreSynched }; -} - -inline bb::world_state::WorldStateStatusSummary world_state_status_summary_from_wire( - const wire::WorldStateStatusSummary& w) -{ - return { w.unfinalizedBlockNumber, w.finalizedBlockNumber, w.oldestHistoricalBlock, w.treesAreSynched }; -} - -inline wire::WorldStateStatusFull world_state_status_full_to_wire(const bb::world_state::WorldStateStatusFull& d) -{ - return { .summary = world_state_status_summary_to_wire(d.summary), - .dbStats = world_state_db_stats_to_wire(d.dbStats), - .meta = world_state_meta_to_wire(d.meta) }; -} - -inline bb::world_state::WorldStateStatusFull world_state_status_full_from_wire(const wire::WorldStateStatusFull& w) -{ - return { world_state_status_summary_from_wire(w.summary), - world_state_db_stats_from_wire(w.dbStats), - world_state_meta_from_wire(w.meta) }; -} - } // namespace bb::wsdb diff --git a/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt deleted file mode 100644 index a44babcef490..000000000000 --- a/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -barretenberg_module(world_state crypto_merkle_tree crypto_poseidon2) - -# world_state needs -fPIC because it's linked into the nodejs_module shared library -set_target_properties(world_state_objects PROPERTIES POSITION_INDEPENDENT_CODE ON) diff --git a/barretenberg/cpp/src/barretenberg/world_state/fork.hpp b/barretenberg/cpp/src/barretenberg/world_state/fork.hpp deleted file mode 100644 index 63134bb2b83d..000000000000 --- a/barretenberg/cpp/src/barretenberg/world_state/fork.hpp +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once - -#include "barretenberg/aztec/aztec_hash_policy.hpp" -#include "barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp" -#include "barretenberg/world_state/tree_with_store.hpp" -#include "barretenberg/world_state/types.hpp" -#include -#include - -namespace bb::world_state { - -// Append-only trees (note-hash, L1->L2 message, archive) share the baseline merkle separator. -using AppendOnlyHashPolicy = aztec::AztecMerkleHashPolicy; - -using FrStore = crypto::merkle_tree::ContentAddressedCachedTreeStore; -using FrTree = crypto::merkle_tree::ContentAddressedAppendOnlyTree; - -using NullifierStore = crypto::merkle_tree::ContentAddressedCachedTreeStore; -using NullifierTree = - crypto::merkle_tree::ContentAddressedIndexedTree; - -using PublicDataStore = crypto::merkle_tree::ContentAddressedCachedTreeStore; -using PublicDataTree = - crypto::merkle_tree::ContentAddressedIndexedTree; - -using Tree = std::variant, TreeWithStore, TreeWithStore>; - -struct Fork { - using Id = uint64_t; - using SharedPtr = std::shared_ptr; - Id _forkId; - std::unordered_map _trees; - index_t _blockNumber; -}; -} // namespace bb::world_state diff --git a/barretenberg/cpp/src/barretenberg/world_state/types.cpp b/barretenberg/cpp/src/barretenberg/world_state/types.cpp deleted file mode 100644 index 3be022ed8297..000000000000 --- a/barretenberg/cpp/src/barretenberg/world_state/types.cpp +++ /dev/null @@ -1,4 +0,0 @@ -#include "barretenberg/world_state/types.hpp" - -// getMerkleTreeName now lives in crypto/merkle_tree/merkle_tree_id.cpp and is re-exported into -// bb::world_state via the `using namespace bb::crypto::merkle_tree` in types.hpp. diff --git a/barretenberg/cpp/src/barretenberg/wsdb/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/wsdb/CMakeLists.txt deleted file mode 100644 index ca88f8e4b9ab..000000000000 --- a/barretenberg/cpp/src/barretenberg/wsdb/CMakeLists.txt +++ /dev/null @@ -1,88 +0,0 @@ -# IPC client library (used by AVM simulator to talk to aztec-wsdb). -# Sources are generated by ipc-codegen from wsdb_schema.jsonc. The custom -# command below wires generation into ninja so clients and servers only -# consume the generated output directory. -set(WSDB_SCHEMA ${CMAKE_CURRENT_SOURCE_DIR}/wsdb_schema.jsonc) -set(WSDB_GEN_PARENT ${CMAKE_CURRENT_SOURCE_DIR}) -set(WSDB_GEN_DIR ${WSDB_GEN_PARENT}/generated) -set(WSDB_GEN_OUTPUTS - ${WSDB_GEN_DIR}/wsdb_ipc_client.cpp - ${WSDB_GEN_DIR}/wsdb_ipc_client.hpp - ${WSDB_GEN_DIR}/wsdb_ipc_server.hpp - ${WSDB_GEN_DIR}/wsdb_types.hpp - ${WSDB_GEN_DIR}/ipc_codegen/msgpack_adaptor.hpp - ${WSDB_GEN_DIR}/ipc_codegen/named_union.hpp - ${WSDB_GEN_DIR}/ipc_codegen/throw.hpp -) -set(IPC_CODEGEN_DIR ${CMAKE_SOURCE_DIR}/../../ipc-codegen) -file(GLOB_RECURSE IPC_CODEGEN_SRC - ${IPC_CODEGEN_DIR}/src/*.ts - ${IPC_CODEGEN_DIR}/templates/cpp/*.hpp -) -add_custom_command( - OUTPUT ${WSDB_GEN_OUTPUTS} - COMMAND node --experimental-strip-types --experimental-transform-types --no-warnings - ${IPC_CODEGEN_DIR}/src/generate.ts - --schema ${WSDB_SCHEMA} - --lang cpp - --out ${WSDB_GEN_DIR} - --client --server - --cpp-namespace bb::wsdb - --cpp-include-dir barretenberg/wsdb/generated - DEPENDS ${WSDB_SCHEMA} ${IPC_CODEGEN_SRC} - COMMENT "Generating WSDB IPC client + server from wsdb_schema.jsonc" - VERBATIM -) -add_custom_target(wsdb_ipc_generated DEPENDS ${WSDB_GEN_OUTPUTS}) -add_library( - wsdb_ipc_client - STATIC - ${WSDB_GEN_DIR}/wsdb_ipc_client.cpp -) -add_dependencies(wsdb_ipc_client wsdb_ipc_generated) -target_include_directories( - wsdb_ipc_client - PUBLIC - ${WSDB_GEN_DIR} -) -target_link_libraries( - wsdb_ipc_client - PUBLIC - barretenberg - ipc_runtime -) - -# aztec-wsdb binary (standalone world state database server) -add_executable( - aztec-wsdb - main.cpp - cli.cpp - wsdb_handlers.cpp - wsdb_ipc_server.cpp -) -add_dependencies(aztec-wsdb wsdb_ipc_generated) -target_include_directories( - aztec-wsdb - PRIVATE - ${WSDB_GEN_DIR} -) -target_link_libraries( - aztec-wsdb - PRIVATE - barretenberg - world_state - ipc_runtime - env -) -if(ENABLE_STACKTRACES) - target_link_libraries( - aztec-wsdb - PUBLIC - Backward::Interface - ) - target_link_options( - aztec-wsdb - PRIVATE - -ldw -lelf - ) -endif() diff --git a/barretenberg/cpp/src/barretenberg/wsdb/main.cpp b/barretenberg/cpp/src/barretenberg/wsdb/main.cpp deleted file mode 100644 index 8857922c678a..000000000000 --- a/barretenberg/cpp/src/barretenberg/wsdb/main.cpp +++ /dev/null @@ -1,6 +0,0 @@ -#include "barretenberg/wsdb/cli.hpp" - -int main(int argc, char* argv[]) -{ - return bb::wsdb::parse_and_run_wsdb(argc, argv); -} diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_client.hpp b/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_client.hpp deleted file mode 100644 index ed738a4c419b..000000000000 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_client.hpp +++ /dev/null @@ -1,3 +0,0 @@ -#pragma once - -#include "barretenberg/wsdb/generated/wsdb_ipc_client.hpp" diff --git a/bootstrap.sh b/bootstrap.sh index 42e87fc8eb47..0c9b77a0d9c9 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -307,6 +307,7 @@ function install_hooks { #!/usr/bin/env bash set -euo pipefail (cd barretenberg/cpp && ./format.sh staged) +(cd native-packages && ./format.sh staged) ./yarn-project/precommit.sh ./noir/precommit.sh ./noir-projects/precommit.sh @@ -573,7 +574,9 @@ function release { projects=( barretenberg/cpp ipc-runtime - wsdb + native-packages/lmdblib + native-packages/kvdb + native-packages/wsdb barretenberg/ts barretenberg/rust noir diff --git a/ci.sh b/ci.sh index 5a1936abfcf7..b20db20425f9 100755 --- a/ci.sh +++ b/ci.sh @@ -233,14 +233,8 @@ case "$cmd" in grind) # Grind a default of 5 times. export CI_DASHBOARD="local" - export DENOISE=1 - export DENOISE_WIDTH=32 - run() { - JOB_ID=$1 INSTANCE_POSTFIX=$1 ARCH=$2 exec denoise "bootstrap_ec2 './bootstrap.sh $3'" - } - export -f run - seq 1 ${1:-5} | parallel --jobs 100 --termseq 'TERM,10000' --tagstring '{= $_=~s/run (\w+).*/$1/; =}' --line-buffered \ - 'run $USER-x{}-full amd64 ci-full-no-test-cache' + multi_job_run \ + $USER'-x'{1..${1:-5}}'-full amd64 ci-full-no-test-cache' ;; merge-queue) # We perform full runs of all tests on multiple x86, and a single fast run on arm64. diff --git a/ipc-codegen/echo_example/cpp/src/echo_client.cpp b/ipc-codegen/echo_example/cpp/src/echo_client.cpp index ca2dc3751d9e..ca06956cc0c8 100644 --- a/ipc-codegen/echo_example/cpp/src/echo_client.cpp +++ b/ipc-codegen/echo_example/cpp/src/echo_client.cpp @@ -17,8 +17,8 @@ } while (0) namespace { -echo::Fr test_hash(uint8_t base) { - echo::Fr hash{}; +echo::wire::Fr test_hash(uint8_t base) { + echo::wire::Fr hash{}; for (size_t i = 0; i < hash.size(); ++i) { hash[i] = static_cast(base + i); } @@ -74,7 +74,7 @@ int main(int argc, char **argv) { CHECK(resp.treeId == 7, "EchoAliases treeId"); CHECK(resp.hash == hash, "EchoAliases hash"); CHECK(resp.maybeHash == second, "EchoAliases maybeHash"); - CHECK((resp.hashes == std::vector{hash, second}), + CHECK((resp.hashes == std::vector{hash, second}), "EchoAliases hashes"); std::cerr << "echo_client(cpp): EchoAliases OK\n"; } diff --git a/ipc-codegen/echo_example/cpp/src/golden_test.cpp b/ipc-codegen/echo_example/cpp/src/golden_test.cpp index 747317683b3e..0649c4fa79c6 100644 --- a/ipc-codegen/echo_example/cpp/src/golden_test.cpp +++ b/ipc-codegen/echo_example/cpp/src/golden_test.cpp @@ -151,12 +151,12 @@ void check_response(const std::string &dir, const std::string &file, } } -echo::Fr test_hash(uint8_t base) { +echo::wire::Fr test_hash(uint8_t base) { std::array bytes{}; for (size_t i = 0; i < bytes.size(); i++) { bytes[i] = static_cast(base + i); } - return echo::Fr(bytes); + return echo::wire::Fr(bytes); } } // namespace diff --git a/ipc-codegen/src/cpp_codegen.ts b/ipc-codegen/src/cpp_codegen.ts index c1df1e42ca61..c993526f7aa8 100644 --- a/ipc-codegen/src/cpp_codegen.ts +++ b/ipc-codegen/src/cpp_codegen.ts @@ -550,10 +550,10 @@ template struct Bin32Alias { namespace ${ns} { -${aliasDecls} - ${this.opts.wireNamespace ? `namespace ${this.opts.wireNamespace} {` : ""} +${aliasDecls} + ${structs} ${this.opts.wireNamespace ? `} // namespace ${this.opts.wireNamespace}` : ""} diff --git a/native-packages/.clang-format b/native-packages/.clang-format new file mode 100644 index 000000000000..14a6b3bac238 --- /dev/null +++ b/native-packages/.clang-format @@ -0,0 +1,26 @@ +PointerAlignment: Left +ColumnLimit: 120 +IndentWidth: 4 +BinPackArguments: false +BinPackParameters: false +Cpp11BracedListStyle: false +AlwaysBreakAfterReturnType: None +AlwaysBreakAfterDefinitionReturnType: None +PenaltyReturnTypeOnItsOwnLine: 1000000 +BreakConstructorInitializers: BeforeComma +BreakBeforeBraces: Custom +BraceWrapping: + AfterClass: false + AfterEnum: false + AfterFunction: true + AfterNamespace: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + SplitEmptyFunction: false + SplitEmptyRecord: false + SplitEmptyNamespace: false +AllowShortFunctionsOnASingleLine : Inline +SortIncludes: true diff --git a/native-packages/format.sh b/native-packages/format.sh new file mode 100755 index 000000000000..ace483f550c7 --- /dev/null +++ b/native-packages/format.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# Formats C++ across the native-packages (wsdb, lmdblib, kvdb, ...). Mirrors +# barretenberg/cpp/format.sh, but rooted at native-packages/ and using this dir's +# .clang-format. barretenberg's format.sh is scoped to barretenberg/cpp (its +# --relative diff excludes siblings), so native-packages needs its own pass. + +# we have to unset this env var set by git hooks so that the relative paths below work correctly when used inside worktrees +unset GIT_DIR + +function format_files { + if [ -n "$1" ]; then + echo "$1" | parallel -j+0 'clang-format-20 -i {} && sed -i.bak "s/\r$//" {} && rm {}.bak' + fi +} + +if [ "$1" == "staged" ]; then + files=$(git diff-index --diff-filter=d --relative --cached --name-only HEAD | grep -e '\.\(cpp\|hpp\|tcc\)$' | grep -v '/generated/') + if [ -n "$files" ]; then + echo Formatting native-packages staged files... + format_files "$files" + echo "$files" | xargs -r git add + fi +elif [ "$1" == "changed" ]; then + files=$(git diff-index --diff-filter=d --relative --name-only HEAD | grep -e '\.\(cpp\|hpp\|tcc\)$' | grep -v '/generated/') + if [ -n "$files" ]; then + echo Formatting native-packages changed files... + format_files "$files" + fi +elif [ "$1" == "check" ]; then + files=$(find . -type d \( -name build -o -name node_modules \) -prune -o \ + \( -iname '*.hpp' -o -iname '*.cpp' -o -iname '*.tcc' \) -print | grep -v '/generated/') + echo "$files" | parallel -N10 clang-format-20 --dry-run --Werror +elif [ -n "$1" ]; then + files=$(git diff-index --relative --name-only $1 | grep -e '\.\(cpp\|hpp\|tcc\)$' | grep -v '/generated/') + format_files "$files" +else + files=$(find . -type d \( -name build -o -name node_modules \) -prune -o \ + \( -iname '*.hpp' -o -iname '*.cpp' -o -iname '*.tcc' \) -print | grep -v '/generated/') + format_files "$files" +fi diff --git a/wsdb/.gitignore b/native-packages/kvdb/.gitignore similarity index 63% rename from wsdb/.gitignore rename to native-packages/kvdb/.gitignore index 5e538a0ae7de..a0e6495b6ebd 100644 --- a/wsdb/.gitignore +++ b/native-packages/kvdb/.gitignore @@ -1,9 +1,13 @@ -ts/ +cpp/build/ +ts/dest/ +ts/build/ +ts/packages/ +ts/.tsbuildinfo +node_modules +*.log .yarn/* !.yarn/patches !.yarn/plugins !.yarn/releases !.yarn/sdks !.yarn/versions -node_modules -*.log diff --git a/native-packages/kvdb/.rebuild_patterns b/native-packages/kvdb/.rebuild_patterns new file mode 100644 index 000000000000..9cb9042b35bc --- /dev/null +++ b/native-packages/kvdb/.rebuild_patterns @@ -0,0 +1,6 @@ +^native-packages/kvdb/bootstrap\.sh$ +^native-packages/kvdb/package\.json$ +^native-packages/kvdb/\.rebuild_patterns$ +^native-packages/kvdb/cpp/ +^native-packages/kvdb/ts/src/ +^native-packages/lmdblib/ diff --git a/wsdb/.yarnrc.yml b/native-packages/kvdb/.yarnrc.yml similarity index 100% rename from wsdb/.yarnrc.yml rename to native-packages/kvdb/.yarnrc.yml diff --git a/native-packages/kvdb/bootstrap.sh b/native-packages/kvdb/bootstrap.sh new file mode 100755 index 000000000000..179cc2a58f38 --- /dev/null +++ b/native-packages/kvdb/bootstrap.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +source $(git rev-parse --show-toplevel)/ci3/source_bootstrap + +ROOT=$(git rev-parse --show-toplevel) +PKG="$ROOT/native-packages/kvdb" +NAPI_BINARY=nodejs_module.node + +hash=$(hash_str \ + $(../lmdblib/bootstrap.sh hash) \ + $(cache_content_hash .rebuild_patterns)) + +# Build the NAPI addon. kvdb is barretenberg-free: it links only the lmdblib +# sibling package's archives (+ lmdb, node-addon-api). lmdblib is assumed prebuilt. +function build_native { + local build_dir="cpp/build" + CC=$(which clang) CXX=$(which clang++) cmake -S cpp -B "$build_dir" -G Ninja >/dev/null + cmake --build "$build_dir" --target nodejs_module + local target_dir="ts/build/$(arch)-$(os)" + mkdir -p "$target_dir" + cp "$build_dir/lib/$NAPI_BINARY" "$target_dir/$NAPI_BINARY" +} + +function build { + echo_header "kvdb build" + build_native + # Create the per-arch workspace packages BEFORE installing: npm_install_deps does + # a clean `yarn install --immutable` on a cache miss, and the @aztec/kvdb- + # optionalDependencies must resolve to these local workspaces (not npm) or the + # install 404s. + (cd ts && ./scripts/prepare_arch_packages.sh "$(arch)-$(os)=build/$(arch)-$(os)/$NAPI_BINARY") + npm_install_deps + yarn build +} + +function clean { + rm -rf ts/dest ts/build ts/packages ts/node_modules node_modules cpp/build +} + +function release { + build_native + npm_install_deps + yarn build + (cd ts && ./scripts/prepare_arch_packages.sh) + for package_dir in ts/packages/*; do + (cd "$package_dir" && retry "deploy_npm ${REF_NAME#v}") + done + (cd ts && retry "deploy_npm ${REF_NAME#v}") +} + +export -f build_native build clean release + +case "$cmd" in + "") + build + ;; + "hash") + echo "$hash" + ;; + *) + default_cmd_handler "$@" + ;; +esac diff --git a/native-packages/kvdb/cpp/.gitignore b/native-packages/kvdb/cpp/.gitignore new file mode 100644 index 000000000000..61c3bc75a05e --- /dev/null +++ b/native-packages/kvdb/cpp/.gitignore @@ -0,0 +1 @@ +.yarn diff --git a/native-packages/kvdb/cpp/.yarnrc.yml b/native-packages/kvdb/cpp/.yarnrc.yml new file mode 100644 index 000000000000..1d8b263ddcf5 --- /dev/null +++ b/native-packages/kvdb/cpp/.yarnrc.yml @@ -0,0 +1,5 @@ +nodeLinker: node-modules + +npmMinimalAgeGate: 7d + +npmPreapprovedPackages: [] diff --git a/native-packages/kvdb/cpp/CMakeLists.txt b/native-packages/kvdb/cpp/CMakeLists.txt new file mode 100644 index 000000000000..116685f69e7c --- /dev/null +++ b/native-packages/kvdb/cpp/CMakeLists.txt @@ -0,0 +1,96 @@ +# Standalone build for the kvdb Node NAPI addon (nodejs_module.node). +# +# This package owns the kv-store native backend (the LMDBStore NAPI wrapper) that +# barretenberg no longer hosts. It is barretenberg-free: it links only the lmdblib +# sibling package's archives + lmdb + the Node addon runtime. It uses msgpack-c +# directly (no bb serialization framework) and vendors the small message header / +# dispatcher it needs under src/messaging. + +cmake_minimum_required(VERSION 3.24) +project(aztec_kvdb CXX C) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) + +# Default to Release (-O3 -DNDEBUG) to match the prebuilt liblmdblib.a it links: the +# shared lmdblib templated/inline headers must be compiled the same way in both, or +# their inline codegen diverges (ODR). +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif() + +add_definitions(-DNAPI_VERSION=9) + +# --------------------------------------------------------------------------- +# lmdblib sibling package build outputs (the only native dependency). +# --------------------------------------------------------------------------- +get_filename_component(REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../.." ABSOLUTE) +set(LMDBLIB_PKG "${REPO_ROOT}/native-packages/lmdblib") +set(LMDBLIB_SRC "${LMDBLIB_PKG}/cpp/src") +set(LMDBLIB_LIB "${LMDBLIB_PKG}/cpp/build/lib/liblmdblib.a") +set(LMDB_INCLUDE "${LMDBLIB_PKG}/cpp/build/_deps/lmdb-src/libraries/liblmdb") +set(LMDB_LIB "${LMDBLIB_PKG}/cpp/build/lib/liblmdb.a") +# msgpack-c headers (header-only) come transitively from lmdblib's build: lmdblib fetches +# msgpack-c and compiles liblmdblib.a against it, so the wire ABI is single-sourced rather +# than re-fetched here. (Matches how the wsdb package consumes msgpack.) +set(MSGPACK_INCLUDE "${LMDBLIB_PKG}/cpp/build/_deps/msgpack-src/include") + +foreach(artifact "${LMDBLIB_LIB}" "${LMDB_LIB}" "${MSGPACK_INCLUDE}") + if(NOT EXISTS "${artifact}") + message(FATAL_ERROR "Missing prebuilt artifact: ${artifact}\n" + "Build lmdblib first (cd native-packages/lmdblib && ./bootstrap.sh).") + endif() +endforeach() + +# --------------------------------------------------------------------------- +# Node addon-api / node-api headers (resolved from this package's node_modules). +# --------------------------------------------------------------------------- +execute_process( + COMMAND yarn --immutable + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) +execute_process( + COMMAND node -p "require('node-addon-api').include" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_VARIABLE NODE_ADDON_API_DIR +) +execute_process( + COMMAND node -p "require('node-api-headers').include_dir" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_VARIABLE NODE_API_HEADERS_DIR +) +string(REGEX REPLACE "[\r\n\"]" "" NODE_ADDON_API_DIR ${NODE_ADDON_API_DIR}) +string(REGEX REPLACE "[\r\n\"]" "" NODE_API_HEADERS_DIR ${NODE_API_HEADERS_DIR}) + +# libc++/-march=skylake match lmdblib's archive (ABI + arch compatibility). Static +# libc++/libgcc keep the addon self-contained (no libc++.so.1 runtime dependency). +add_compile_definitions(MSGPACK_NO_BOOST MSGPACK_USE_STD_VARIANT_ADAPTOR NO_PAR_ALGOS) +add_compile_options(-march=skylake -fconstexpr-steps=100000000 -fbracket-depth=1024) +add_compile_options(-Wno-deprecated-declarations -Wno-missing-field-initializers) +add_compile_options(-stdlib=libc++) +add_link_options(-stdlib=libc++ -static-libstdc++ -static-libgcc) + +include_directories( + ${CMAKE_CURRENT_SOURCE_DIR}/src + ${LMDBLIB_SRC} + ${MSGPACK_INCLUDE} + ${LMDB_INCLUDE} +) + +set(KVDB_SOURCES + src/init_module.cpp + src/lmdb_store/lmdb_store_wrapper.cpp + src/util/promise.cpp +) +add_library(nodejs_module SHARED ${KVDB_SOURCES}) +set_target_properties(nodejs_module PROPERTIES PREFIX "" SUFFIX ".node") +target_include_directories(nodejs_module PRIVATE ${NODE_API_HEADERS_DIR} ${NODE_ADDON_API_DIR}) + +target_link_libraries(nodejs_module PRIVATE ${LMDBLIB_LIB} ${LMDB_LIB} pthread) + +# On macOS, Node.js N-API symbols are provided by the runtime, not at link time. +if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + target_link_options(nodejs_module PRIVATE "-undefined" "dynamic_lookup") +endif() diff --git a/native-packages/kvdb/cpp/package.json b/native-packages/kvdb/cpp/package.json new file mode 100644 index 000000000000..b30468a65e83 --- /dev/null +++ b/native-packages/kvdb/cpp/package.json @@ -0,0 +1,15 @@ +{ + "name": "nodejs_module", + "private": true, + "version": "0.0.0", + "packageManager": "yarn@4.13.0", + "dependencies": { + "node-addon-api": "^8.0.0", + "node-api-headers": "^1.1.0" + }, + "binary": { + "napi_versions": [ + 9 + ] + } +} diff --git a/native-packages/kvdb/cpp/src/init_module.cpp b/native-packages/kvdb/cpp/src/init_module.cpp new file mode 100644 index 000000000000..1cd88633e002 --- /dev/null +++ b/native-packages/kvdb/cpp/src/init_module.cpp @@ -0,0 +1,11 @@ +#include "lmdb_store/lmdb_store_wrapper.hpp" +#include "napi.h" + +Napi::Object Init(Napi::Env env, Napi::Object exports) { + exports.Set(Napi::String::New(env, "LMDBStore"), + azteclabs::kvdb::lmdb_store::LMDBStoreWrapper::get_class(env)); + return exports; +} + +// NOLINTNEXTLINE +NODE_API_MODULE(addon, Init) diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/lmdb_store/lmdb_store_message.hpp b/native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_message.hpp similarity index 65% rename from barretenberg/cpp/src/barretenberg/nodejs_module/lmdb_store/lmdb_store_message.hpp rename to native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_message.hpp index b63bd1849f7d..497b78669a50 100644 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/lmdb_store/lmdb_store_message.hpp +++ b/native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_message.hpp @@ -1,15 +1,15 @@ #pragma once -#include "barretenberg/lmdblib/types.hpp" -#include "barretenberg/messaging/header.hpp" -#include "barretenberg/serialize/msgpack.hpp" +#include "lmdblib/types.hpp" +#include "messaging/header.hpp" +#include #include "msgpack/adaptor/define_decl.hpp" #include #include #include -namespace bb::nodejs::lmdb_store { +namespace azteclabs::kvdb::lmdb_store { -using namespace bb::messaging; +using namespace azteclabs::kvdb::messaging; enum LMDBStoreMessageType { OPEN_DATABASE = FIRST_APP_MSG_TYPE, @@ -33,43 +33,43 @@ enum LMDBStoreMessageType { struct OpenDatabaseRequest { std::string db; std::optional uniqueKeys; - SERIALIZATION_FIELDS(db, uniqueKeys); + MSGPACK_DEFINE_MAP(db, uniqueKeys); }; struct GetRequest { lmdblib::KeysVector keys; std::string db; - SERIALIZATION_FIELDS(keys, db); + MSGPACK_DEFINE_MAP(keys, db); }; struct GetResponse { lmdblib::OptionalValuesVector values; - SERIALIZATION_FIELDS(values); + MSGPACK_DEFINE_MAP(values); }; struct HasRequest { // std::map> entries; lmdblib::KeyOptionalValuesVector entries; std::string db; - SERIALIZATION_FIELDS(entries, db); + MSGPACK_DEFINE_MAP(entries, db); }; struct HasResponse { // std::map exists; std::vector exists; - SERIALIZATION_FIELDS(exists); + MSGPACK_DEFINE_MAP(exists); }; struct Batch { lmdblib::KeyDupValuesVector addEntries; lmdblib::KeyOptionalValuesVector removeEntries; - SERIALIZATION_FIELDS(addEntries, removeEntries); + MSGPACK_DEFINE_MAP(addEntries, removeEntries); }; struct BatchRequest { std::map batches; - SERIALIZATION_FIELDS(batches); + MSGPACK_DEFINE_MAP(batches); }; struct StartCursorRequest { @@ -78,67 +78,67 @@ struct StartCursorRequest { std::optional count; std::optional onePage; std::string db; - SERIALIZATION_FIELDS(key, reverse, count, onePage, db); + MSGPACK_DEFINE_MAP(key, reverse, count, onePage, db); }; struct StartCursorResponse { std::optional cursor; lmdblib::KeyDupValuesVector entries; - SERIALIZATION_FIELDS(cursor, entries); + MSGPACK_DEFINE_MAP(cursor, entries); }; struct AdvanceCursorRequest { uint64_t cursor; std::optional count; - SERIALIZATION_FIELDS(cursor, count); + MSGPACK_DEFINE_MAP(cursor, count); }; struct AdvanceCursorCountRequest { uint64_t cursor; lmdblib::Key endKey; - SERIALIZATION_FIELDS(cursor, endKey); + MSGPACK_DEFINE_MAP(cursor, endKey); }; struct CloseCursorRequest { uint64_t cursor; - SERIALIZATION_FIELDS(cursor); + MSGPACK_DEFINE_MAP(cursor); }; struct AdvanceCursorResponse { lmdblib::KeyDupValuesVector entries; bool done; - SERIALIZATION_FIELDS(entries, done); + MSGPACK_DEFINE_MAP(entries, done); }; struct AdvanceCursorCountResponse { uint64_t count; bool done; - SERIALIZATION_FIELDS(count, done); + MSGPACK_DEFINE_MAP(count, done); }; struct BoolResponse { bool ok; - SERIALIZATION_FIELDS(ok); + MSGPACK_DEFINE_MAP(ok); }; struct BatchResponse { uint64_t durationNs; - SERIALIZATION_FIELDS(durationNs); + MSGPACK_DEFINE_MAP(durationNs); }; struct StatsResponse { std::vector stats; uint64_t dbMapSizeBytes; uint64_t dbPhysicalFileSizeBytes; - SERIALIZATION_FIELDS(stats, dbMapSizeBytes, dbPhysicalFileSizeBytes); + MSGPACK_DEFINE_MAP(stats, dbMapSizeBytes, dbPhysicalFileSizeBytes); }; struct CopyStoreRequest { std::string dstPath; std::optional compact; - SERIALIZATION_FIELDS(dstPath, compact); + MSGPACK_DEFINE_MAP(dstPath, compact); }; -} // namespace bb::nodejs::lmdb_store +} // namespace azteclabs::kvdb::lmdb_store -MSGPACK_ADD_ENUM(bb::nodejs::lmdb_store::LMDBStoreMessageType) +MSGPACK_ADD_ENUM(azteclabs::kvdb::lmdb_store::LMDBStoreMessageType) diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/lmdb_store/lmdb_store_wrapper.cpp b/native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_wrapper.cpp similarity index 95% rename from barretenberg/cpp/src/barretenberg/nodejs_module/lmdb_store/lmdb_store_wrapper.cpp rename to native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_wrapper.cpp index b891f0e3f115..7436e37d4a7e 100644 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/lmdb_store/lmdb_store_wrapper.cpp +++ b/native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_wrapper.cpp @@ -1,7 +1,7 @@ -#include "barretenberg/nodejs_module/lmdb_store/lmdb_store_wrapper.hpp" -#include "barretenberg/lmdblib/lmdb_store.hpp" -#include "barretenberg/lmdblib/types.hpp" -#include "barretenberg/nodejs_module/lmdb_store/lmdb_store_message.hpp" +#include "lmdb_store/lmdb_store_wrapper.hpp" +#include "lmdblib/lmdb_store.hpp" +#include "lmdblib/types.hpp" +#include "lmdb_store/lmdb_store_message.hpp" #include "napi.h" #include #include @@ -11,8 +11,8 @@ #include #include -using namespace bb::nodejs; -using namespace bb::nodejs::lmdb_store; +using namespace azteclabs::kvdb; +using namespace azteclabs::kvdb::lmdb_store; const uint64_t DEFAULT_MAP_SIZE = 1024UL * 1024; const uint64_t DEFAULT_MAX_READERS = 16; @@ -113,7 +113,7 @@ void LMDBStoreWrapper::verify_store() const if (_store) { return; } - throw std::runtime_error(format("LMDB store unavailable, was close already called?")); + throw std::runtime_error(azteclabs::lmdblib::format("LMDB store unavailable, was close already called?")); } BoolResponse LMDBStoreWrapper::open_database(const OpenDatabaseRequest& req) @@ -281,7 +281,7 @@ BoolResponse LMDBStoreWrapper::copy_store(const CopyStoreRequest& req) return { true }; } -std::pair LMDBStoreWrapper::_advance_cursor(const lmdblib::LMDBCursor& cursor, +std::pair LMDBStoreWrapper::_advance_cursor(const lmdblib::LMDBCursor& cursor, bool reverse, uint64_t page_size) { diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/lmdb_store/lmdb_store_wrapper.hpp b/native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_wrapper.hpp similarity index 79% rename from barretenberg/cpp/src/barretenberg/nodejs_module/lmdb_store/lmdb_store_wrapper.hpp rename to native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_wrapper.hpp index 4fea656edc5b..33a5eea41392 100644 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/lmdb_store/lmdb_store_wrapper.hpp +++ b/native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_wrapper.hpp @@ -1,18 +1,18 @@ #pragma once -#include "barretenberg/lmdblib/lmdb_cursor.hpp" -#include "barretenberg/lmdblib/lmdb_store.hpp" -#include "barretenberg/lmdblib/types.hpp" -#include "barretenberg/messaging/dispatcher.hpp" -#include "barretenberg/messaging/header.hpp" -#include "barretenberg/nodejs_module/lmdb_store/lmdb_store_message.hpp" -#include "barretenberg/nodejs_module/util/message_processor.hpp" +#include "lmdblib/lmdb_cursor.hpp" +#include "lmdblib/lmdb_store.hpp" +#include "lmdblib/types.hpp" +#include "messaging/dispatcher.hpp" +#include "messaging/header.hpp" +#include "lmdb_store/lmdb_store_message.hpp" +#include "util/message_processor.hpp" #include #include #include #include -namespace bb::nodejs::lmdb_store { +namespace azteclabs::kvdb::lmdb_store { struct CursorData { lmdblib::LMDBCursor::SharedPtr cursor; @@ -38,7 +38,7 @@ class LMDBStoreWrapper : public Napi::ObjectWrap { std::mutex _cursor_mutex; std::unordered_map _cursors; - bb::nodejs::AsyncMessageProcessor _msg_processor; + azteclabs::kvdb::AsyncMessageProcessor _msg_processor; void verify_store() const; @@ -69,4 +69,4 @@ class LMDBStoreWrapper : public Napi::ObjectWrap { const lmdblib::Key& end_key); }; -} // namespace bb::nodejs::lmdb_store +} // namespace azteclabs::kvdb::lmdb_store diff --git a/native-packages/kvdb/cpp/src/messaging/dispatcher.hpp b/native-packages/kvdb/cpp/src/messaging/dispatcher.hpp new file mode 100644 index 000000000000..48759d306068 --- /dev/null +++ b/native-packages/kvdb/cpp/src/messaging/dispatcher.hpp @@ -0,0 +1,59 @@ +#pragma once + +#include "messaging/header.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace azteclabs::kvdb::messaging { + +using message_handler = + std::function; +struct MessageHandler { + bool unique; + message_handler handler; +}; + +class MessageDispatcher { +private: + std::unordered_map message_handlers; + mutable std::shared_mutex mutex; + +public: + MessageDispatcher() = default; + + bool on_new_data(msgpack::object &obj, msgpack::sbuffer &buffer) const { + azteclabs::kvdb::messaging::HeaderOnlyMessage header; + obj.convert(header); + + auto iter = message_handlers.find(header.msgType); + if (iter == message_handlers.end()) { + throw std::runtime_error("No registered handler for message of type " + + std::to_string(header.msgType)); + } + + // If the msg type has been marked as 'unique' then we need to give it + // exclusive execution context + if (iter->second.unique) { + std::unique_lock lock(mutex); + return (iter->second.handler)(obj, buffer); + } + std::shared_lock lock(mutex); + return (iter->second.handler)(obj, buffer); + } + + void register_target(uint32_t msgType, const message_handler &handler, + bool unique = false) { + MessageHandler msg_handler{unique, handler}; + message_handlers.insert({msgType, msg_handler}); + } +}; + +} // namespace azteclabs::kvdb::messaging diff --git a/native-packages/kvdb/cpp/src/messaging/header.hpp b/native-packages/kvdb/cpp/src/messaging/header.hpp new file mode 100644 index 000000000000..f27ab138852d --- /dev/null +++ b/native-packages/kvdb/cpp/src/messaging/header.hpp @@ -0,0 +1,52 @@ +#pragma once +#include +#include +#include + +namespace azteclabs::kvdb::messaging { + +enum SystemMsgTypes { TERMINATE = 0, PING = 1, PONG = 2 }; + +const uint32_t FIRST_APP_MSG_TYPE = 100; + +struct MsgHeader { + uint32_t messageId; // Unique Id for the message + uint32_t + requestId; // Id of the message this is responding too (may not be used) + + MSGPACK_DEFINE_MAP(messageId, requestId); + + MsgHeader() = default; + + MsgHeader(uint32_t reqId) : requestId(reqId) {} + + MsgHeader(uint32_t msgId, uint32_t reqId) + : messageId(msgId), requestId(reqId) {} +}; + +struct HeaderOnlyMessage { + uint32_t msgType; + MsgHeader header; + + HeaderOnlyMessage(uint32_t type, MsgHeader &hdr) + : msgType(type), header(hdr) {} + + HeaderOnlyMessage() = default; + + MSGPACK_DEFINE_MAP(msgType, header); +}; + +template struct TypedMessage { + uint32_t msgType; + MsgHeader header; + T value; + + TypedMessage(uint32_t type, MsgHeader &hdr, const T &val) + : msgType(type), header(hdr), value(val) {} + + TypedMessage() = default; + + MSGPACK_DEFINE_MAP(msgType, header, value); +}; + +} // namespace azteclabs::kvdb::messaging diff --git a/native-packages/kvdb/cpp/src/util/async_op.hpp b/native-packages/kvdb/cpp/src/util/async_op.hpp new file mode 100644 index 000000000000..a814ce016e67 --- /dev/null +++ b/native-packages/kvdb/cpp/src/util/async_op.hpp @@ -0,0 +1,200 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#ifndef _WIN32 +#include +#endif + +namespace azteclabs::kvdb { + +using async_fn = std::function; + +/** + * @brief Encapsulatest some work that can be done off the JavaScript main thread + * + * This class takes a Deferred instance (i.e. a Promise to JS), execute some work in a separate thread, and then report + * back on the result. The async execution _must not_ touch the JS environment. Everything that's needed to complete the + * work must be copied into memory owned by the C++ code. The same has to be done when reporting back the result: keep + * the result in memory owned by the C++ code and copy it back to the JS environment in the OnOK/OnError methods. + * + * OnOK/OnError will be called on the main JS thread, so it's safe to interact with the JS environment there. + * + * Instances of this class are managed by the NodeJS environment and execute on a libuv thread. + * Docs + * . - https://github.com/nodejs/node-addon-api/blob/cc06369aa4dd29e585600b8b47839c1297df962d/doc/async_worker.md + * . - https://nodejs.github.io/node-addon-examples/special-topics/asyncworker + */ +class AsyncOperation : public Napi::AsyncWorker { + public: + AsyncOperation(Napi::Env env, std::shared_ptr deferred, async_fn fn) + : Napi::AsyncWorker(env) + , _fn(std::move(fn)) + , _deferred(std::move(deferred)) + {} + + AsyncOperation(const AsyncOperation&) = delete; + AsyncOperation& operator=(const AsyncOperation&) = delete; + AsyncOperation(AsyncOperation&&) = delete; + AsyncOperation& operator=(AsyncOperation&&) = delete; + + ~AsyncOperation() override = default; + + void Execute() override + { + try { + _fn(_result); + } catch (const std::exception& e) { + SetError(e.what()); + } catch (...) { + // Catch any other exception type that's not derived from std::exception + // This ensures the promise is always rejected rather than leaving it hanging + SetError("Unknown exception occurred during async operation"); + } + } + + void OnOK() override + { + auto buf = Napi::Buffer::Copy(Env(), _result.data(), _result.size()); + _deferred->Resolve(buf); + } + void OnError(const Napi::Error& e) override { _deferred->Reject(e.Value()); } + + private: + async_fn _fn; + std::shared_ptr _deferred; + msgpack::sbuffer _result; +}; + +/** + * @brief Runs work on a dedicated std::thread instead of the libuv thread pool. + * + * Unlike AsyncOperation (which uses Napi::AsyncWorker and occupies a libuv thread), + * this class spawns a new OS thread for each operation. This prevents AVM simulations + * from exhausting the libuv thread pool, which would deadlock when C++ callbacks need + * to invoke JS functions that themselves require libuv threads (e.g., LMDB reads). + * + * The completion callback (resolve/reject) is posted back to the JS main thread via + * a Napi::ThreadSafeFunction, so the event loop returns immediately after launch + * and is woken up only when the work is done. + * + * Prevent use-after-free: the TSFN callback runs asynchronously on the JS thread + * (napi_tsfn_blocking only blocks on queue insertion, NOT on callback completion). + * Both the worker thread lambda and the callback capture a shared_ptr to keep the + * object alive until both are done. + * + * Usage: `ThreadedAsyncOperation::Run(env, deferred, fn);` + */ +class ThreadedAsyncOperation : public std::enable_shared_from_this { + public: + ThreadedAsyncOperation(Napi::Env env, std::shared_ptr deferred, async_fn fn) + : _fn(std::move(fn)) + , _deferred(std::move(deferred)) + { + auto dummy = Napi::Function::New(env, [](const Napi::CallbackInfo&) {}); + _completion_tsfn = Napi::ThreadSafeFunction::New(env, dummy, "ThreadedAsyncOpComplete", 0, 1); + } + + ThreadedAsyncOperation(const ThreadedAsyncOperation&) = delete; + ThreadedAsyncOperation& operator=(const ThreadedAsyncOperation&) = delete; + ThreadedAsyncOperation(ThreadedAsyncOperation&&) = delete; + ThreadedAsyncOperation& operator=(ThreadedAsyncOperation&&) = delete; + + ~ThreadedAsyncOperation() = default; + + static void Run(Napi::Env env, std::shared_ptr deferred, async_fn fn) + { + auto op = std::make_shared(env, std::move(deferred), std::move(fn)); + op->Queue(); + } + + private: + // AVM simulation call chains are deep. Non-main threads get a 512 KB default stack on + // macOS versus 8 MB on Linux, so a default std::thread overflows its stack-guard page and + // aborts with SIGBUS on macOS arm64. The libuv pool that AsyncOperation runs on sizes its + // threads from RLIMIT_STACK, which is why that path never hit this. Pin a generous stack so + // the worker has the same headroom on every platform. + static constexpr size_t WORKER_STACK_SIZE = 32UL * 1024 * 1024; + + void Queue() + { + auto self = shared_from_this(); + launch_detached_with_large_stack([self]() { + try { + self->_fn(self->_result); + self->_success = true; + } catch (const std::exception& e) { + self->_error = e.what(); + self->_success = false; + } catch (...) { + self->_error = "Unknown exception occurred during threaded async operation"; + self->_success = false; + } + + // Post completion to the JS main thread. The callback captures `self` + // (shared_ptr) so the object stays alive until the callback runs. + // napi_tsfn_blocking only blocks on queue insertion, not on callback + // completion, so we cannot use raw pointers here. + self->_completion_tsfn.BlockingCall([self](Napi::Env env, Napi::Function /*js_callback*/) { + if (self->_success) { + auto buf = Napi::Buffer::Copy(env, self->_result.data(), self->_result.size()); + self->_deferred->Resolve(buf); + } else { + auto error = Napi::Error::New(env, self->_error); + self->_deferred->Reject(error.Value()); + } + self->_completion_tsfn.Release(); + }); + }); + } + + // Launch `work` on a detached OS thread with an explicitly large stack (see WORKER_STACK_SIZE). + // std::thread cannot set a stack size, so use pthreads where available and fall back to a + // default-stack std::thread only if pthread creation is unavailable or fails. + static void launch_detached_with_large_stack(std::function work) + { +#ifndef _WIN32 + pthread_attr_t attr; + if (pthread_attr_init(&attr) == 0) { + pthread_attr_setstacksize(&attr, WORKER_STACK_SIZE); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + + auto* heap_work = new std::function(std::move(work)); + pthread_t tid; + int rc = pthread_create( + &tid, + &attr, + [](void* arg) -> void* { + std::unique_ptr> fn(static_cast*>(arg)); + (*fn)(); + return nullptr; + }, + heap_work); + pthread_attr_destroy(&attr); + + if (rc == 0) { + return; + } + + // pthread_create failed; reclaim the work and fall back to a default std::thread. + std::unique_ptr> reclaimed(heap_work); + work = std::move(*reclaimed); + } +#endif + std::thread(std::move(work)).detach(); + } + + async_fn _fn; + std::shared_ptr _deferred; + Napi::ThreadSafeFunction _completion_tsfn; + msgpack::sbuffer _result; + bool _success = false; + std::string _error; +}; + +} // namespace azteclabs::kvdb diff --git a/native-packages/kvdb/cpp/src/util/message_processor.hpp b/native-packages/kvdb/cpp/src/util/message_processor.hpp new file mode 100644 index 000000000000..ede1fa8f4d68 --- /dev/null +++ b/native-packages/kvdb/cpp/src/util/message_processor.hpp @@ -0,0 +1,128 @@ +#pragma once + +#include "messaging/dispatcher.hpp" +#include "messaging/header.hpp" +#include "util/async_op.hpp" +#include "napi.h" +#include + +namespace azteclabs::kvdb { + +class AsyncMessageProcessor { + public: + template + void register_handler(uint32_t msgType, T* self, R (T::*handler)() const, bool unique = false) + { + register_handler(msgType, self, handler, unique); + } + + template + void register_handler(uint32_t msgType, T* self, R (T::*handler)(), bool unique = false) + { + _register_handler( + msgType, [=](auto, const msgpack::object&) { return (self->*handler)(); }, unique); + } + + template + void register_handler(uint32_t msgType, T* self, R (T::*handler)(const P&) const, bool unique = false) + { + register_handler(msgType, self, handler, unique); + } + + template + void register_handler(uint32_t msgType, T* self, R (T::*handler)(const P&), bool unique = false) + { + _register_handler, R>( + msgType, + [=](const messaging::TypedMessage

& req, const msgpack::object&) { return (self->*handler)(req.value); }, + unique); + } + + template + void register_handler(uint32_t msgType, + T* self, + R (T::*handler)(const P&, const msgpack::object&) const, + bool unique = false) + { + register_handler(msgType, self, handler, unique); + } + + template + void register_handler(uint32_t msgType, + T* self, + R (T::*handler)(const P&, const msgpack::object&), + bool unique = false) + { + _register_handler, R>( + msgType, + [=](const messaging::TypedMessage

& req, const msgpack::object& obj) { + return (self->*handler)(req.value, obj); + }, + unique); + } + + Napi::Promise process_message(const Napi::CallbackInfo& info) + { + Napi::Env env = info.Env(); + // keep this in a shared pointer so that AsyncOperation can resolve/reject the promise once the execution is + // complete on an separate thread + auto deferred = std::make_shared(env); + + if (!open) { + deferred->Reject(Napi::TypeError::New(env, "Message processor is closed").Value()); + } else if (info.Length() < 1) { + deferred->Reject(Napi::TypeError::New(env, "Wrong number of arguments").Value()); + } else if (!info[0].IsBuffer()) { + deferred->Reject(Napi::TypeError::New(env, "Argument must be a buffer").Value()); + } else { + auto buffer = info[0].As>(); + size_t length = buffer.Length(); + // we mustn't access the Napi::Env outside of this top-level function + // so copy the data to a variable we own + // and make it a shared pointer so that it doesn't get destroyed as soon as we exit this code block + auto data = std::make_shared>(length); + std::copy_n(buffer.Data(), length, data->data()); + + auto* op = new azteclabs::kvdb::AsyncOperation(env, deferred, [data, this, length](msgpack::sbuffer& buf) { + msgpack::object_handle obj_handle = msgpack::unpack(data->data(), length); + msgpack::object obj = obj_handle.get(); + dispatcher.on_new_data(obj, buf); + }); + + // Napi is now responsible for destroying this object + op->Queue(); + } + + return deferred->Promise(); + } + + void close() { open = false; } + + private: + azteclabs::kvdb::messaging::MessageDispatcher dispatcher; + std::atomic_bool open = true; + + template + void _register_handler(uint32_t msgType, + const std::function& fn, + bool unique = false) + { + dispatcher.register_target( + msgType, + [=](msgpack::object& obj, msgpack::sbuffer& buffer) { + P req_msg; + obj.convert(req_msg); + + R response = fn(req_msg, obj); + + azteclabs::kvdb::messaging::MsgHeader header(req_msg.header.messageId); + azteclabs::kvdb::messaging::TypedMessage resp_msg(msgType, header, response); + msgpack::pack(buffer, resp_msg); + + return true; + }, + unique); + } +}; + +} // namespace azteclabs::kvdb diff --git a/native-packages/kvdb/cpp/src/util/promise.cpp b/native-packages/kvdb/cpp/src/util/promise.cpp new file mode 100644 index 000000000000..00e1d6995470 --- /dev/null +++ b/native-packages/kvdb/cpp/src/util/promise.cpp @@ -0,0 +1,12 @@ + +#include "napi.h" + +namespace azteclabs::kvdb { + +Napi::Promise promise_reject(const Napi::Env& env, const Napi::Value& err) +{ + auto def = Napi::Promise::Deferred::New(env); + def.Reject(err); + return def.Promise(); +} +} // namespace azteclabs::kvdb diff --git a/native-packages/kvdb/cpp/src/util/promise.hpp b/native-packages/kvdb/cpp/src/util/promise.hpp new file mode 100644 index 000000000000..5cf90ee3c815 --- /dev/null +++ b/native-packages/kvdb/cpp/src/util/promise.hpp @@ -0,0 +1,9 @@ + +#pragma once + +#include "messaging/dispatcher.hpp" +#include "napi.h" + +namespace azteclabs::kvdb { +Napi::Promise promise_reject(const Napi::Env& env, const Napi::Value& err); +} diff --git a/native-packages/kvdb/cpp/yarn.lock b/native-packages/kvdb/cpp/yarn.lock new file mode 100644 index 000000000000..ed3111738283 --- /dev/null +++ b/native-packages/kvdb/cpp/yarn.lock @@ -0,0 +1,579 @@ +# This file is generated by running "yarn install" inside your project. +# Manual changes might be lost - proceed with caution! + +__metadata: + version: 8 + cacheKey: 10c0 + +"@isaacs/fs-minipass@npm:^4.0.0": + version: 4.0.1 + resolution: "@isaacs/fs-minipass@npm:4.0.1" + dependencies: + minipass: "npm:^7.0.4" + checksum: 10c0/c25b6dc1598790d5b55c0947a9b7d111cfa92594db5296c3b907e2f533c033666f692a3939eadac17b1c7c40d362d0b0635dc874cbfe3e70db7c2b07cc97a5d2 + languageName: node + linkType: hard + +"@npmcli/agent@npm:^4.0.0": + version: 4.0.0 + resolution: "@npmcli/agent@npm:4.0.0" + dependencies: + agent-base: "npm:^7.1.0" + http-proxy-agent: "npm:^7.0.0" + https-proxy-agent: "npm:^7.0.1" + lru-cache: "npm:^11.2.1" + socks-proxy-agent: "npm:^8.0.3" + checksum: 10c0/f7b5ce0f3dd42c3f8c6546e8433573d8049f67ef11ec22aa4704bc41483122f68bf97752e06302c455ead667af5cb753e6a09bff06632bc465c1cfd4c4b75a53 + languageName: node + linkType: hard + +"@npmcli/fs@npm:^5.0.0": + version: 5.0.0 + resolution: "@npmcli/fs@npm:5.0.0" + dependencies: + semver: "npm:^7.3.5" + checksum: 10c0/26e376d780f60ff16e874a0ac9bc3399186846baae0b6e1352286385ac134d900cc5dafaded77f38d77f86898fc923ae1cee9d7399f0275b1aa24878915d722b + languageName: node + linkType: hard + +"abbrev@npm:^4.0.0": + version: 4.0.0 + resolution: "abbrev@npm:4.0.0" + checksum: 10c0/b4cc16935235e80702fc90192e349e32f8ef0ed151ef506aa78c81a7c455ec18375c4125414b99f84b2e055199d66383e787675f0bcd87da7a4dbd59f9eac1d5 + languageName: node + linkType: hard + +"agent-base@npm:^7.1.0, agent-base@npm:^7.1.2": + version: 7.1.4 + resolution: "agent-base@npm:7.1.4" + checksum: 10c0/c2c9ab7599692d594b6a161559ada307b7a624fa4c7b03e3afdb5a5e31cd0e53269115b620fcab024c5ac6a6f37fa5eb2e004f076ad30f5f7e6b8b671f7b35fe + languageName: node + linkType: hard + +"balanced-match@npm:^4.0.2": + version: 4.0.4 + resolution: "balanced-match@npm:4.0.4" + checksum: 10c0/07e86102a3eb2ee2a6a1a89164f29d0dbaebd28f2ca3f5ca786f36b8b23d9e417eb3be45a4acf754f837be5ac0a2317de90d3fcb7f4f4dc95720a1f36b26a17b + languageName: node + linkType: hard + +"brace-expansion@npm:^5.0.2": + version: 5.0.4 + resolution: "brace-expansion@npm:5.0.4" + dependencies: + balanced-match: "npm:^4.0.2" + checksum: 10c0/359cbcfa80b2eb914ca1f3440e92313fbfe7919ee6b274c35db55bec555aded69dac5ee78f102cec90c35f98c20fa43d10936d0cd9978158823c249257e1643a + languageName: node + linkType: hard + +"cacache@npm:^20.0.1": + version: 20.0.3 + resolution: "cacache@npm:20.0.3" + dependencies: + "@npmcli/fs": "npm:^5.0.0" + fs-minipass: "npm:^3.0.0" + glob: "npm:^13.0.0" + lru-cache: "npm:^11.1.0" + minipass: "npm:^7.0.3" + minipass-collect: "npm:^2.0.1" + minipass-flush: "npm:^1.0.5" + minipass-pipeline: "npm:^1.2.4" + p-map: "npm:^7.0.2" + ssri: "npm:^13.0.0" + unique-filename: "npm:^5.0.0" + checksum: 10c0/c7da1ca694d20e8f8aedabd21dc11518f809a7d2b59aa76a1fc655db5a9e62379e465c157ddd2afe34b19230808882288effa6911b2de26a088a6d5645123462 + languageName: node + linkType: hard + +"chownr@npm:^3.0.0": + version: 3.0.0 + resolution: "chownr@npm:3.0.0" + checksum: 10c0/43925b87700f7e3893296c8e9c56cc58f926411cce3a6e5898136daaf08f08b9a8eb76d37d3267e707d0dcc17aed2e2ebdf5848c0c3ce95cf910a919935c1b10 + languageName: node + linkType: hard + +"debug@npm:4, debug@npm:^4.3.4": + version: 4.4.3 + resolution: "debug@npm:4.4.3" + dependencies: + ms: "npm:^2.1.3" + peerDependenciesMeta: + supports-color: + optional: true + checksum: 10c0/d79136ec6c83ecbefd0f6a5593da6a9c91ec4d7ddc4b54c883d6e71ec9accb5f67a1a5e96d00a328196b5b5c86d365e98d8a3a70856aaf16b4e7b1985e67f5a6 + languageName: node + linkType: hard + +"encoding@npm:^0.1.13": + version: 0.1.13 + resolution: "encoding@npm:0.1.13" + dependencies: + iconv-lite: "npm:^0.6.2" + checksum: 10c0/36d938712ff00fe1f4bac88b43bcffb5930c1efa57bbcdca9d67e1d9d6c57cfb1200fb01efe0f3109b2ce99b231f90779532814a81370a1bd3274a0f58585039 + languageName: node + linkType: hard + +"env-paths@npm:^2.2.0": + version: 2.2.1 + resolution: "env-paths@npm:2.2.1" + checksum: 10c0/285325677bf00e30845e330eec32894f5105529db97496ee3f598478e50f008c5352a41a30e5e72ec9de8a542b5a570b85699cd63bd2bc646dbcb9f311d83bc4 + languageName: node + linkType: hard + +"err-code@npm:^2.0.2": + version: 2.0.3 + resolution: "err-code@npm:2.0.3" + checksum: 10c0/b642f7b4dd4a376e954947550a3065a9ece6733ab8e51ad80db727aaae0817c2e99b02a97a3d6cecc648a97848305e728289cf312d09af395403a90c9d4d8a66 + languageName: node + linkType: hard + +"exponential-backoff@npm:^3.1.1": + version: 3.1.3 + resolution: "exponential-backoff@npm:3.1.3" + checksum: 10c0/77e3ae682b7b1f4972f563c6dbcd2b0d54ac679e62d5d32f3e5085feba20483cf28bd505543f520e287a56d4d55a28d7874299941faf637e779a1aa5994d1267 + languageName: node + linkType: hard + +"fdir@npm:^6.5.0": + version: 6.5.0 + resolution: "fdir@npm:6.5.0" + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + checksum: 10c0/e345083c4306b3aed6cb8ec551e26c36bab5c511e99ea4576a16750ddc8d3240e63826cc624f5ae17ad4dc82e68a253213b60d556c11bfad064b7607847ed07f + languageName: node + linkType: hard + +"fs-minipass@npm:^3.0.0": + version: 3.0.3 + resolution: "fs-minipass@npm:3.0.3" + dependencies: + minipass: "npm:^7.0.3" + checksum: 10c0/63e80da2ff9b621e2cb1596abcb9207f1cf82b968b116ccd7b959e3323144cce7fb141462200971c38bbf2ecca51695069db45265705bed09a7cd93ae5b89f94 + languageName: node + linkType: hard + +"glob@npm:^13.0.0": + version: 13.0.2 + resolution: "glob@npm:13.0.2" + dependencies: + minimatch: "npm:^10.1.2" + minipass: "npm:^7.1.2" + path-scurry: "npm:^2.0.0" + checksum: 10c0/3d4b09efa922c4cba9be6d5b9efae14384e7422aeb886eb35fba8a94820b8281474b8d3f16927127fb1a0c8580e18fc00e3fda03c8dc31fa0af3ba918edeeb04 + languageName: node + linkType: hard + +"graceful-fs@npm:^4.2.6": + version: 4.2.11 + resolution: "graceful-fs@npm:4.2.11" + checksum: 10c0/386d011a553e02bc594ac2ca0bd6d9e4c22d7fa8cfbfc448a6d148c59ea881b092db9dbe3547ae4b88e55f1b01f7c4a2ecc53b310c042793e63aa44cf6c257f2 + languageName: node + linkType: hard + +"http-cache-semantics@npm:^4.1.1": + version: 4.2.0 + resolution: "http-cache-semantics@npm:4.2.0" + checksum: 10c0/45b66a945cf13ec2d1f29432277201313babf4a01d9e52f44b31ca923434083afeca03f18417f599c9ab3d0e7b618ceb21257542338b57c54b710463b4a53e37 + languageName: node + linkType: hard + +"http-proxy-agent@npm:^7.0.0": + version: 7.0.2 + resolution: "http-proxy-agent@npm:7.0.2" + dependencies: + agent-base: "npm:^7.1.0" + debug: "npm:^4.3.4" + checksum: 10c0/4207b06a4580fb85dd6dff521f0abf6db517489e70863dca1a0291daa7f2d3d2d6015a57bd702af068ea5cf9f1f6ff72314f5f5b4228d299c0904135d2aef921 + languageName: node + linkType: hard + +"https-proxy-agent@npm:^7.0.1": + version: 7.0.6 + resolution: "https-proxy-agent@npm:7.0.6" + dependencies: + agent-base: "npm:^7.1.2" + debug: "npm:4" + checksum: 10c0/f729219bc735edb621fa30e6e84e60ee5d00802b8247aac0d7b79b0bd6d4b3294737a337b93b86a0bd9e68099d031858a39260c976dc14cdbba238ba1f8779ac + languageName: node + linkType: hard + +"iconv-lite@npm:^0.6.2": + version: 0.6.3 + resolution: "iconv-lite@npm:0.6.3" + dependencies: + safer-buffer: "npm:>= 2.1.2 < 3.0.0" + checksum: 10c0/98102bc66b33fcf5ac044099d1257ba0b7ad5e3ccd3221f34dd508ab4070edff183276221684e1e0555b145fce0850c9f7d2b60a9fcac50fbb4ea0d6e845a3b1 + languageName: node + linkType: hard + +"imurmurhash@npm:^0.1.4": + version: 0.1.4 + resolution: "imurmurhash@npm:0.1.4" + checksum: 10c0/8b51313850dd33605c6c9d3fd9638b714f4c4c40250cff658209f30d40da60f78992fb2df5dabee4acf589a6a82bbc79ad5486550754bd9ec4e3fc0d4a57d6a6 + languageName: node + linkType: hard + +"ip-address@npm:^10.0.1": + version: 10.1.0 + resolution: "ip-address@npm:10.1.0" + checksum: 10c0/0103516cfa93f6433b3bd7333fa876eb21263912329bfa47010af5e16934eeeff86f3d2ae700a3744a137839ddfad62b900c7a445607884a49b5d1e32a3d7566 + languageName: node + linkType: hard + +"isexe@npm:^4.0.0": + version: 4.0.0 + resolution: "isexe@npm:4.0.0" + checksum: 10c0/5884815115bceac452877659a9c7726382531592f43dc29e5d48b7c4100661aed54018cb90bd36cb2eaeba521092570769167acbb95c18d39afdccbcca06c5ce + languageName: node + linkType: hard + +"lru-cache@npm:^11.0.0, lru-cache@npm:^11.1.0, lru-cache@npm:^11.2.1": + version: 11.2.6 + resolution: "lru-cache@npm:11.2.6" + checksum: 10c0/73bbffb298760e71b2bfe8ebc16a311c6a60ceddbba919cfedfd8635c2d125fbfb5a39b71818200e67973b11f8d59c5a9e31d6f90722e340e90393663a66e5cd + languageName: node + linkType: hard + +"make-fetch-happen@npm:^15.0.0": + version: 15.0.3 + resolution: "make-fetch-happen@npm:15.0.3" + dependencies: + "@npmcli/agent": "npm:^4.0.0" + cacache: "npm:^20.0.1" + http-cache-semantics: "npm:^4.1.1" + minipass: "npm:^7.0.2" + minipass-fetch: "npm:^5.0.0" + minipass-flush: "npm:^1.0.5" + minipass-pipeline: "npm:^1.2.4" + negotiator: "npm:^1.0.0" + proc-log: "npm:^6.0.0" + promise-retry: "npm:^2.0.1" + ssri: "npm:^13.0.0" + checksum: 10c0/525f74915660be60b616bcbd267c4a5b59481b073ba125e45c9c3a041bb1a47a2bd0ae79d028eb6f5f95bf9851a4158423f5068539c3093621abb64027e8e461 + languageName: node + linkType: hard + +"minimatch@npm:^10.1.2": + version: 10.2.4 + resolution: "minimatch@npm:10.2.4" + dependencies: + brace-expansion: "npm:^5.0.2" + checksum: 10c0/35f3dfb7b99b51efd46afd378486889f590e7efb10e0f6a10ba6800428cf65c9a8dedb74427d0570b318d749b543dc4e85f06d46d2858bc8cac7e1eb49a95945 + languageName: node + linkType: hard + +"minipass-collect@npm:^2.0.1": + version: 2.0.1 + resolution: "minipass-collect@npm:2.0.1" + dependencies: + minipass: "npm:^7.0.3" + checksum: 10c0/5167e73f62bb74cc5019594709c77e6a742051a647fe9499abf03c71dca75515b7959d67a764bdc4f8b361cf897fbf25e2d9869ee039203ed45240f48b9aa06e + languageName: node + linkType: hard + +"minipass-fetch@npm:^5.0.0": + version: 5.0.1 + resolution: "minipass-fetch@npm:5.0.1" + dependencies: + encoding: "npm:^0.1.13" + minipass: "npm:^7.0.3" + minipass-sized: "npm:^2.0.0" + minizlib: "npm:^3.0.1" + dependenciesMeta: + encoding: + optional: true + checksum: 10c0/50bcf48c9841ebb25e29a2817468595219c72cfffc7c175a1d7327843c8bef9b72cb01778f46df7eca695dfe47ab98e6167af4cb026ddd80f660842919a5193c + languageName: node + linkType: hard + +"minipass-flush@npm:^1.0.5": + version: 1.0.5 + resolution: "minipass-flush@npm:1.0.5" + dependencies: + minipass: "npm:^3.0.0" + checksum: 10c0/2a51b63feb799d2bb34669205eee7c0eaf9dce01883261a5b77410c9408aa447e478efd191b4de6fc1101e796ff5892f8443ef20d9544385819093dbb32d36bd + languageName: node + linkType: hard + +"minipass-pipeline@npm:^1.2.4": + version: 1.2.4 + resolution: "minipass-pipeline@npm:1.2.4" + dependencies: + minipass: "npm:^3.0.0" + checksum: 10c0/cbda57cea20b140b797505dc2cac71581a70b3247b84480c1fed5ca5ba46c25ecc25f68bfc9e6dcb1a6e9017dab5c7ada5eab73ad4f0a49d84e35093e0c643f2 + languageName: node + linkType: hard + +"minipass-sized@npm:^2.0.0": + version: 2.0.0 + resolution: "minipass-sized@npm:2.0.0" + dependencies: + minipass: "npm:^7.1.2" + checksum: 10c0/f9201696a6f6d68610d04c9c83e3d2e5cb9c026aae1c8cbf7e17f386105cb79c1bb088dbc21bf0b1eb4f3fb5df384fd1e7aa3bf1f33868c416ae8c8a92679db8 + languageName: node + linkType: hard + +"minipass@npm:^3.0.0": + version: 3.3.6 + resolution: "minipass@npm:3.3.6" + dependencies: + yallist: "npm:^4.0.0" + checksum: 10c0/a114746943afa1dbbca8249e706d1d38b85ed1298b530f5808ce51f8e9e941962e2a5ad2e00eae7dd21d8a4aae6586a66d4216d1a259385e9d0358f0c1eba16c + languageName: node + linkType: hard + +"minipass@npm:^7.0.2, minipass@npm:^7.0.3, minipass@npm:^7.0.4, minipass@npm:^7.1.2": + version: 7.1.2 + resolution: "minipass@npm:7.1.2" + checksum: 10c0/b0fd20bb9fb56e5fa9a8bfac539e8915ae07430a619e4b86ff71f5fc757ef3924b23b2c4230393af1eda647ed3d75739e4e0acb250a6b1eb277cf7f8fe449557 + languageName: node + linkType: hard + +"minizlib@npm:^3.0.1, minizlib@npm:^3.1.0": + version: 3.1.0 + resolution: "minizlib@npm:3.1.0" + dependencies: + minipass: "npm:^7.1.2" + checksum: 10c0/5aad75ab0090b8266069c9aabe582c021ae53eb33c6c691054a13a45db3b4f91a7fb1bd79151e6b4e9e9a86727b522527c0a06ec7d45206b745d54cd3097bcec + languageName: node + linkType: hard + +"ms@npm:^2.1.3": + version: 2.1.3 + resolution: "ms@npm:2.1.3" + checksum: 10c0/d924b57e7312b3b63ad21fc5b3dc0af5e78d61a1fc7cfb5457edaf26326bf62be5307cc87ffb6862ef1c2b33b0233cdb5d4f01c4c958cc0d660948b65a287a48 + languageName: node + linkType: hard + +"negotiator@npm:^1.0.0": + version: 1.0.0 + resolution: "negotiator@npm:1.0.0" + checksum: 10c0/4c559dd52669ea48e1914f9d634227c561221dd54734070791f999c52ed0ff36e437b2e07d5c1f6e32909fc625fe46491c16e4a8f0572567d4dd15c3a4fda04b + languageName: node + linkType: hard + +"node-addon-api@npm:^8.0.0": + version: 8.0.0 + resolution: "node-addon-api@npm:8.0.0" + dependencies: + node-gyp: "npm:latest" + checksum: 10c0/20eb231362cc07c62d9839164473744d985be5d82685214f3750d990d9f61ef366e0ba112a766c925d640ed29b2a500b83568e895dc2444dcd5db01e615aac2b + languageName: node + linkType: hard + +"node-api-headers@npm:^1.1.0": + version: 1.1.0 + resolution: "node-api-headers@npm:1.1.0" + checksum: 10c0/7806d71077348ea199034e8c90a9147038d37fcccc1b85717e48c095fe31783a4f909f5daced4506e6cbce93fba91220bb3fc8626ee0640d26de9860f6500174 + languageName: node + linkType: hard + +"node-gyp@npm:latest": + version: 12.2.0 + resolution: "node-gyp@npm:12.2.0" + dependencies: + env-paths: "npm:^2.2.0" + exponential-backoff: "npm:^3.1.1" + graceful-fs: "npm:^4.2.6" + make-fetch-happen: "npm:^15.0.0" + nopt: "npm:^9.0.0" + proc-log: "npm:^6.0.0" + semver: "npm:^7.3.5" + tar: "npm:^7.5.4" + tinyglobby: "npm:^0.2.12" + which: "npm:^6.0.0" + bin: + node-gyp: bin/node-gyp.js + checksum: 10c0/3ed046746a5a7d90950cd8b0547332b06598443f31fe213ef4332a7174c7b7d259e1704835feda79b87d3f02e59d7791842aac60642ede4396ab25fdf0f8f759 + languageName: node + linkType: hard + +"nodejs_module@workspace:.": + version: 0.0.0-use.local + resolution: "nodejs_module@workspace:." + dependencies: + node-addon-api: "npm:^8.0.0" + node-api-headers: "npm:^1.1.0" + languageName: unknown + linkType: soft + +"nopt@npm:^9.0.0": + version: 9.0.0 + resolution: "nopt@npm:9.0.0" + dependencies: + abbrev: "npm:^4.0.0" + bin: + nopt: bin/nopt.js + checksum: 10c0/1822eb6f9b020ef6f7a7516d7b64a8036e09666ea55ac40416c36e4b2b343122c3cff0e2f085675f53de1d2db99a2a89a60ccea1d120bcd6a5347bf6ceb4a7fd + languageName: node + linkType: hard + +"p-map@npm:^7.0.2": + version: 7.0.4 + resolution: "p-map@npm:7.0.4" + checksum: 10c0/a5030935d3cb2919d7e89454d1ce82141e6f9955413658b8c9403cfe379283770ed3048146b44cde168aa9e8c716505f196d5689db0ae3ce9a71521a2fef3abd + languageName: node + linkType: hard + +"path-scurry@npm:^2.0.0": + version: 2.0.1 + resolution: "path-scurry@npm:2.0.1" + dependencies: + lru-cache: "npm:^11.0.0" + minipass: "npm:^7.1.2" + checksum: 10c0/2a16ed0e81fbc43513e245aa5763354e25e787dab0d539581a6c3f0f967461a159ed6236b2559de23aa5b88e7dc32b469b6c47568833dd142a4b24b4f5cd2620 + languageName: node + linkType: hard + +"picomatch@npm:^4.0.3": + version: 4.0.3 + resolution: "picomatch@npm:4.0.3" + checksum: 10c0/9582c951e95eebee5434f59e426cddd228a7b97a0161a375aed4be244bd3fe8e3a31b846808ea14ef2c8a2527a6eeab7b3946a67d5979e81694654f939473ae2 + languageName: node + linkType: hard + +"proc-log@npm:^6.0.0": + version: 6.1.0 + resolution: "proc-log@npm:6.1.0" + checksum: 10c0/4f178d4062733ead9d71a9b1ab24ebcecdfe2250916a5b1555f04fe2eda972a0ec76fbaa8df1ad9c02707add6749219d118a4fc46dc56bdfe4dde4b47d80bb82 + languageName: node + linkType: hard + +"promise-retry@npm:^2.0.1": + version: 2.0.1 + resolution: "promise-retry@npm:2.0.1" + dependencies: + err-code: "npm:^2.0.2" + retry: "npm:^0.12.0" + checksum: 10c0/9c7045a1a2928094b5b9b15336dcd2a7b1c052f674550df63cc3f36cd44028e5080448175b6f6ca32b642de81150f5e7b1a98b728f15cb069f2dd60ac2616b96 + languageName: node + linkType: hard + +"retry@npm:^0.12.0": + version: 0.12.0 + resolution: "retry@npm:0.12.0" + checksum: 10c0/59933e8501727ba13ad73ef4a04d5280b3717fd650408460c987392efe9d7be2040778ed8ebe933c5cbd63da3dcc37919c141ef8af0a54a6e4fca5a2af177bfe + languageName: node + linkType: hard + +"safer-buffer@npm:>= 2.1.2 < 3.0.0": + version: 2.1.2 + resolution: "safer-buffer@npm:2.1.2" + checksum: 10c0/7e3c8b2e88a1841c9671094bbaeebd94448111dd90a81a1f606f3f67708a6ec57763b3b47f06da09fc6054193e0e6709e77325415dc8422b04497a8070fa02d4 + languageName: node + linkType: hard + +"semver@npm:^7.3.5": + version: 7.7.4 + resolution: "semver@npm:7.7.4" + bin: + semver: bin/semver.js + checksum: 10c0/5215ad0234e2845d4ea5bb9d836d42b03499546ddafb12075566899fc617f68794bb6f146076b6881d755de17d6c6cc73372555879ec7dce2c2feee947866ad2 + languageName: node + linkType: hard + +"smart-buffer@npm:^4.2.0": + version: 4.2.0 + resolution: "smart-buffer@npm:4.2.0" + checksum: 10c0/a16775323e1404dd43fabafe7460be13a471e021637bc7889468eb45ce6a6b207261f454e4e530a19500cc962c4cc5348583520843b363f4193cee5c00e1e539 + languageName: node + linkType: hard + +"socks-proxy-agent@npm:^8.0.3": + version: 8.0.5 + resolution: "socks-proxy-agent@npm:8.0.5" + dependencies: + agent-base: "npm:^7.1.2" + debug: "npm:^4.3.4" + socks: "npm:^2.8.3" + checksum: 10c0/5d2c6cecba6821389aabf18728325730504bf9bb1d9e342e7987a5d13badd7a98838cc9a55b8ed3cb866ad37cc23e1086f09c4d72d93105ce9dfe76330e9d2a6 + languageName: node + linkType: hard + +"socks@npm:^2.8.3": + version: 2.8.7 + resolution: "socks@npm:2.8.7" + dependencies: + ip-address: "npm:^10.0.1" + smart-buffer: "npm:^4.2.0" + checksum: 10c0/2805a43a1c4bcf9ebf6e018268d87b32b32b06fbbc1f9282573583acc155860dc361500f89c73bfbb157caa1b4ac78059eac0ef15d1811eb0ca75e0bdadbc9d2 + languageName: node + linkType: hard + +"ssri@npm:^13.0.0": + version: 13.0.1 + resolution: "ssri@npm:13.0.1" + dependencies: + minipass: "npm:^7.0.3" + checksum: 10c0/cf6408a18676c57ff2ed06b8a20dc64bb3e748e5c7e095332e6aecaa2b8422b1e94a739a8453bf65156a8a47afe23757ba4ab52d3ea3b62322dc40875763e17a + languageName: node + linkType: hard + +"tar@npm:^7.5.4": + version: 7.5.11 + resolution: "tar@npm:7.5.11" + dependencies: + "@isaacs/fs-minipass": "npm:^4.0.0" + chownr: "npm:^3.0.0" + minipass: "npm:^7.1.2" + minizlib: "npm:^3.1.0" + yallist: "npm:^5.0.0" + checksum: 10c0/b6bb420550ef50ef23356018155e956cd83282c97b6128d8d5cfe5740c57582d806a244b2ef0bf686a74ce526babe8b8b9061527623e935e850008d86d838929 + languageName: node + linkType: hard + +"tinyglobby@npm:^0.2.12": + version: 0.2.15 + resolution: "tinyglobby@npm:0.2.15" + dependencies: + fdir: "npm:^6.5.0" + picomatch: "npm:^4.0.3" + checksum: 10c0/869c31490d0d88eedb8305d178d4c75e7463e820df5a9b9d388291daf93e8b1eb5de1dad1c1e139767e4269fe75f3b10d5009b2cc14db96ff98986920a186844 + languageName: node + linkType: hard + +"unique-filename@npm:^5.0.0": + version: 5.0.0 + resolution: "unique-filename@npm:5.0.0" + dependencies: + unique-slug: "npm:^6.0.0" + checksum: 10c0/afb897e9cf4c2fb622ea716f7c2bb462001928fc5f437972213afdf1cc32101a230c0f1e9d96fc91ee5185eca0f2feb34127145874975f347be52eb91d6ccc2c + languageName: node + linkType: hard + +"unique-slug@npm:^6.0.0": + version: 6.0.0 + resolution: "unique-slug@npm:6.0.0" + dependencies: + imurmurhash: "npm:^0.1.4" + checksum: 10c0/da7ade4cb04eb33ad0499861f82fe95ce9c7c878b7139dc54d140ecfb6a6541c18a5c8dac16188b8b379fe62c0c1f1b710814baac910cde5f4fec06212126c6a + languageName: node + linkType: hard + +"which@npm:^6.0.0": + version: 6.0.1 + resolution: "which@npm:6.0.1" + dependencies: + isexe: "npm:^4.0.0" + bin: + node-which: bin/which.js + checksum: 10c0/7e710e54ea36d2d6183bee2f9caa27a3b47b9baf8dee55a199b736fcf85eab3b9df7556fca3d02b50af7f3dfba5ea3a45644189836df06267df457e354da66d5 + languageName: node + linkType: hard + +"yallist@npm:^4.0.0": + version: 4.0.0 + resolution: "yallist@npm:4.0.0" + checksum: 10c0/2286b5e8dbfe22204ab66e2ef5cc9bbb1e55dfc873bbe0d568aa943eb255d131890dfd5bf243637273d31119b870f49c18fcde2c6ffbb7a7a092b870dc90625a + languageName: node + linkType: hard + +"yallist@npm:^5.0.0": + version: 5.0.0 + resolution: "yallist@npm:5.0.0" + checksum: 10c0/a499c81ce6d4a1d260d4ea0f6d49ab4da09681e32c3f0472dee16667ed69d01dae63a3b81745a24bd78476ec4fcf856114cb4896ace738e01da34b2c42235416 + languageName: node + linkType: hard diff --git a/native-packages/kvdb/package.json b/native-packages/kvdb/package.json new file mode 100644 index 000000000000..6dabbfc25130 --- /dev/null +++ b/native-packages/kvdb/package.json @@ -0,0 +1,13 @@ +{ + "name": "@aztec/kvdb-packages", + "packageManager": "yarn@4.13.0", + "private": true, + "scripts": { + "build": "yarn workspace @aztec/kvdb build", + "clean": "yarn workspaces foreach -A -p -v run clean" + }, + "workspaces": [ + "ts", + "ts/packages/*" + ] +} diff --git a/native-packages/kvdb/ts/package.json b/native-packages/kvdb/ts/package.json new file mode 100644 index 000000000000..f92a5163ec1d --- /dev/null +++ b/native-packages/kvdb/ts/package.json @@ -0,0 +1,33 @@ +{ + "name": "@aztec/kvdb", + "version": "0.1.0", + "type": "module", + "exports": { + ".": { + "types": "./dest/index.d.ts", + "default": "./dest/index.js" + } + }, + "files": [ + "dest/", + "README.md" + ], + "scripts": { + "clean": "rm -rf dest .tsbuildinfo", + "build": "tsc -p tsconfig.json", + "prepare_arch_packages": "./scripts/prepare_arch_packages.sh" + }, + "dependencies": { + "tslib": "^2.4.0" + }, + "optionalDependencies": { + "@aztec/kvdb-darwin-arm64": "0.1.0", + "@aztec/kvdb-darwin-x64": "0.1.0", + "@aztec/kvdb-linux-arm64": "0.1.0", + "@aztec/kvdb-linux-x64": "0.1.0" + }, + "devDependencies": { + "@types/node": "^22.15.17", + "typescript": "^5.3.3" + } +} diff --git a/native-packages/kvdb/ts/scripts/prepare_arch_packages.sh b/native-packages/kvdb/ts/scripts/prepare_arch_packages.sh new file mode 100755 index 000000000000..701b6f3e0f8e --- /dev/null +++ b/native-packages/kvdb/ts/scripts/prepare_arch_packages.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +declare -A PLATFORMS=( + ["amd64-linux"]="linux-x64 linux x64" + ["arm64-linux"]="linux-arm64 linux arm64" + ["amd64-macos"]="darwin-x64 darwin x64" + ["arm64-macos"]="darwin-arm64 darwin arm64" +) + +NAPI_BINARY="nodejs_module.node" +version=$(node -p "require('./package.json').version") + +declare -A BINARIES=() +for arg in "$@"; do + case "$arg" in + *=*) + key="${arg%%=*}" + value="${arg#*=}" + BINARIES["$key"]="$value" + ;; + *) + echo "Usage: npm run prepare_arch_packages -- [= ...]" >&2 + echo "Platforms: linux-x64, linux-arm64, darwin-x64, darwin-arm64" >&2 + exit 1 + ;; + esac +done + +for build_dir in "${!PLATFORMS[@]}"; do + read -r suffix os cpu <<< "${PLATFORMS[$build_dir]}" + pkg_name="@aztec/kvdb-${suffix}" + out_dir="packages/kvdb-${suffix}" + binary_path="${BINARIES[$suffix]:-${BINARIES[$build_dir]:-}}" + + if [ -z "$binary_path" ]; then + binary_path="build/${build_dir}/${NAPI_BINARY}" + fi + + # Always create the arch workspace stub so the workspace set (and lockfile) is + # stable on single-arch builds. The os/cpu fields make npm install only the + # matching package at runtime; the binary is copied in only where available. + rm -rf "${out_dir}" + mkdir -p "${out_dir}" + if [ -f "$binary_path" ]; then + cp "$binary_path" "${out_dir}/${NAPI_BINARY}" + else + echo "No binary for ${pkg_name} (${binary_path}); creating stub workspace package." + fi + + cat > "${out_dir}/package.json" < = { + "x86_64-linux": "@aztec/kvdb-linux-x64", + "x86_64-darwin": "@aztec/kvdb-darwin-x64", + "aarch64-linux": "@aztec/kvdb-linux-arm64", + "aarch64-darwin": "@aztec/kvdb-darwin-arm64", +}; + +const NAPI_BINARY = "nodejs_module.node"; + +function currentDir(): string { + return path.dirname(fileURLToPath(import.meta.url)); +} + +function detectPlatform(): Platform | null { + if (process.arch === "x64" && process.platform === "linux") + return "x86_64-linux"; + if (process.arch === "x64" && process.platform === "darwin") + return "x86_64-darwin"; + if (process.arch === "arm64" && process.platform === "linux") + return "aarch64-linux"; + if (process.arch === "arm64" && process.platform === "darwin") + return "aarch64-darwin"; + return null; +} + +function findArchPackageDir(platform: Platform): string | null { + const packageName = PLATFORM_TO_PACKAGE[platform]; + try { + const require = createRequire(import.meta.url); + return path.dirname(require.resolve(packageName + "/package.json")); + } catch { + const siblingPackageDir = path.join( + currentDir(), + "..", + "packages", + packageName.split("/").pop()!, + ); + return fs.existsSync(path.join(siblingPackageDir, "package.json")) + ? siblingPackageDir + : null; + } +} + +/** + * Locate the kvdb NAPI addon (nodejs_module.node) for the current platform. + * @param customPath Optional explicit path to the addon (overrides detection). + * @returns Absolute path to the addon, or null if not found. + */ +export function findNapiBinary(customPath?: string): string | null { + if (customPath) { + return fs.existsSync(customPath) ? path.resolve(customPath) : null; + } + + const envPath = process.env.AZTEC_KVDB_NAPI_PATH; + if (envPath) { + return fs.existsSync(envPath) ? path.resolve(envPath) : null; + } + + const platform = detectPlatform(); + if (!platform) { + return null; + } + + const archDir = findArchPackageDir(platform); + if (archDir) { + const candidate = path.join(archDir, NAPI_BINARY); + if (fs.existsSync(candidate)) { + return candidate; + } + } + + return null; +} diff --git a/native-packages/kvdb/ts/tsconfig.json b/native-packages/kvdb/ts/tsconfig.json new file mode 100644 index 000000000000..18049f35db9c --- /dev/null +++ b/native-packages/kvdb/ts/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "declaration": true, + "declarationMap": true, + "composite": true, + "outDir": "dest", + "rootDir": "src", + "tsBuildInfoFile": ".tsbuildinfo", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src/**/*.ts"] +} diff --git a/native-packages/kvdb/yarn.lock b/native-packages/kvdb/yarn.lock new file mode 100644 index 000000000000..23b6685c9d03 --- /dev/null +++ b/native-packages/kvdb/yarn.lock @@ -0,0 +1,102 @@ +# This file is generated by running "yarn install" inside your project. +# Manual changes might be lost - proceed with caution! + +__metadata: + version: 8 + cacheKey: 10c0 + +"@aztec/kvdb-darwin-arm64@npm:0.1.0, @aztec/kvdb-darwin-arm64@workspace:ts/packages/kvdb-darwin-arm64": + version: 0.0.0-use.local + resolution: "@aztec/kvdb-darwin-arm64@workspace:ts/packages/kvdb-darwin-arm64" + languageName: unknown + linkType: soft + +"@aztec/kvdb-darwin-x64@npm:0.1.0, @aztec/kvdb-darwin-x64@workspace:ts/packages/kvdb-darwin-x64": + version: 0.0.0-use.local + resolution: "@aztec/kvdb-darwin-x64@workspace:ts/packages/kvdb-darwin-x64" + languageName: unknown + linkType: soft + +"@aztec/kvdb-linux-arm64@npm:0.1.0, @aztec/kvdb-linux-arm64@workspace:ts/packages/kvdb-linux-arm64": + version: 0.0.0-use.local + resolution: "@aztec/kvdb-linux-arm64@workspace:ts/packages/kvdb-linux-arm64" + languageName: unknown + linkType: soft + +"@aztec/kvdb-linux-x64@npm:0.1.0, @aztec/kvdb-linux-x64@workspace:ts/packages/kvdb-linux-x64": + version: 0.0.0-use.local + resolution: "@aztec/kvdb-linux-x64@workspace:ts/packages/kvdb-linux-x64" + languageName: unknown + linkType: soft + +"@aztec/kvdb-packages@workspace:.": + version: 0.0.0-use.local + resolution: "@aztec/kvdb-packages@workspace:." + languageName: unknown + linkType: soft + +"@aztec/kvdb@workspace:ts": + version: 0.0.0-use.local + resolution: "@aztec/kvdb@workspace:ts" + dependencies: + "@aztec/kvdb-darwin-arm64": "npm:0.1.0" + "@aztec/kvdb-darwin-x64": "npm:0.1.0" + "@aztec/kvdb-linux-arm64": "npm:0.1.0" + "@aztec/kvdb-linux-x64": "npm:0.1.0" + "@types/node": "npm:^22.15.17" + tslib: "npm:^2.4.0" + typescript: "npm:^5.3.3" + dependenciesMeta: + "@aztec/kvdb-darwin-arm64": + optional: true + "@aztec/kvdb-darwin-x64": + optional: true + "@aztec/kvdb-linux-arm64": + optional: true + "@aztec/kvdb-linux-x64": + optional: true + languageName: unknown + linkType: soft + +"@types/node@npm:^22.15.17": + version: 22.20.0 + resolution: "@types/node@npm:22.20.0" + dependencies: + undici-types: "npm:~6.21.0" + checksum: 10c0/55d78223205bd5f81f043d71b7a5c8d8854b9ef44ef81291680943adb27fa5ba1f092658c87183d5bc8cf6baf6a57b81dad966eb3afa452cc301a615b6d9b20e + languageName: node + linkType: hard + +"tslib@npm:^2.4.0": + version: 2.8.1 + resolution: "tslib@npm:2.8.1" + checksum: 10c0/9c4759110a19c53f992d9aae23aac5ced636e99887b51b9e61def52611732872ff7668757d4e4c61f19691e36f4da981cd9485e869b4a7408d689f6bf1f14e62 + languageName: node + linkType: hard + +"typescript@npm:^5.3.3": + version: 5.9.3 + resolution: "typescript@npm:5.9.3" + bin: + tsc: bin/tsc + tsserver: bin/tsserver + checksum: 10c0/6bd7552ce39f97e711db5aa048f6f9995b53f1c52f7d8667c1abdc1700c68a76a308f579cd309ce6b53646deb4e9a1be7c813a93baaf0a28ccd536a30270e1c5 + languageName: node + linkType: hard + +"typescript@patch:typescript@npm%3A^5.3.3#optional!builtin": + version: 5.9.3 + resolution: "typescript@patch:typescript@npm%3A5.9.3#optional!builtin::version=5.9.3&hash=5786d5" + bin: + tsc: bin/tsc + tsserver: bin/tsserver + checksum: 10c0/ad09fdf7a756814dce65bc60c1657b40d44451346858eea230e10f2e95a289d9183b6e32e5c11e95acc0ccc214b4f36289dcad4bf1886b0adb84d711d336a430 + languageName: node + linkType: hard + +"undici-types@npm:~6.21.0": + version: 6.21.0 + resolution: "undici-types@npm:6.21.0" + checksum: 10c0/c01ed51829b10aa72fc3ce64b747f8e74ae9b60eafa19a7b46ef624403508a54c526ffab06a14a26b3120d055e1104d7abe7c9017e83ced038ea5cf52f8d5e04 + languageName: node + linkType: hard diff --git a/native-packages/lmdblib/.gitignore b/native-packages/lmdblib/.gitignore new file mode 100644 index 000000000000..568bb61b59d2 --- /dev/null +++ b/native-packages/lmdblib/.gitignore @@ -0,0 +1 @@ +cpp/build/ diff --git a/native-packages/lmdblib/.rebuild_patterns b/native-packages/lmdblib/.rebuild_patterns new file mode 100644 index 000000000000..9c628b1c31f5 --- /dev/null +++ b/native-packages/lmdblib/.rebuild_patterns @@ -0,0 +1,3 @@ +^native-packages/lmdblib/bootstrap\.sh$ +^native-packages/lmdblib/\.rebuild_patterns$ +^native-packages/lmdblib/cpp/ diff --git a/native-packages/lmdblib/bootstrap.sh b/native-packages/lmdblib/bootstrap.sh new file mode 100755 index 000000000000..6160d5986998 --- /dev/null +++ b/native-packages/lmdblib/bootstrap.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +source $(git rev-parse --show-toplevel)/ci3/source_bootstrap + +ROOT=$(git rev-parse --show-toplevel) +PKG="$ROOT/native-packages/lmdblib" + +hash=$(hash_str \ + $(cache_content_hash .rebuild_patterns)) + +# Build liblmdblib.a + liblmdb.a. lmdblib is barretenberg-free: it builds against +# its own deps (lmdb, msgpack-c) only. Both archives are consumed by the kvdb NAPI +# addon and the wsdb service. +function build_native { + local build_dir="cpp/build" + CC=$(which clang) CXX=$(which clang++) cmake -S cpp -B "$build_dir" -G Ninja >/dev/null + cmake --build "$build_dir" --target lmdblib lmdblib_tests +} + +function build { + echo_header "lmdblib build" + build_native +} + +# Emit test commands for the CI test engine (one self-contained gtest binary). +function test_cmds { + echo "$hash:CPUS=2:TIMEOUT=120s native-packages/lmdblib/cpp/build/lmdblib_tests" +} + +# Manual: build then run the tests directly. +function test { + echo_header "lmdblib test" + build + test_cmds | filter_test_cmds | parallelize +} + +function clean { + rm -rf cpp/build +} + +export -f build_native build test_cmds test clean + +case "$cmd" in + "") + build + ;; + "hash") + echo "$hash" + ;; + *) + default_cmd_handler "$@" + ;; +esac diff --git a/native-packages/lmdblib/cpp/CMakeLists.txt b/native-packages/lmdblib/cpp/CMakeLists.txt new file mode 100644 index 000000000000..13137c21dd43 --- /dev/null +++ b/native-packages/lmdblib/cpp/CMakeLists.txt @@ -0,0 +1,107 @@ +# lmdblib — a small C++ wrapper around LMDB. +# +# Builds two static archives that the kvdb NAPI addon and the wsdb service link: +# - liblmdb.a : the LMDB C library (built from source, pinned commit) +# - liblmdblib.a : the lmdblib C++ wrapper objects +# +# It is self-contained: it vendors LMDB and msgpack-c (header-only) via FetchContent. + +cmake_minimum_required(VERSION 3.24) +# C is enabled to compile the LMDB C sources. +project(aztec_lmdblib CXX C) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) + +# Default to Release (-O3 -DNDEBUG). lmdblib's templated/inline headers (serialise_key, +# value_cmp, the msgpack adaptors) are compiled into both liblmdblib.a and into every +# consumer that includes them (the kvdb addon, the wsdb service); matching the consumers' +# Release build keeps that inline codegen identical and avoids ODR divergence. +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif() + +# --------------------------------------------------------------------------- +# Third-party dependencies (fetched at configure time). +# --------------------------------------------------------------------------- +include(FetchContent) + +# LMDB: a plain-Makefile C library (no CMake). Fetch the source and compile the two +# translation units ourselves so they use this project's toolchain and flags. +FetchContent_Declare( + lmdb + GIT_REPOSITORY https://github.com/LMDB/lmdb.git + GIT_TAG ddd0a773e2f44d38e4e31ec9ed81af81f4e4ccbb +) +FetchContent_MakeAvailable(lmdb) +set(LMDB_DIR ${lmdb_SOURCE_DIR}/libraries/liblmdb) +add_library(lmdb STATIC ${LMDB_DIR}/mdb.c ${LMDB_DIR}/midl.c) +target_include_directories(lmdb PUBLIC ${LMDB_DIR}) + +# msgpack-c: header-only. Fetch the source and use its include directory directly +# (we don't build msgpack-c's own CMake targets/tests). +FetchContent_Declare( + msgpack + GIT_REPOSITORY https://github.com/msgpack/msgpack-c.git + GIT_TAG cpp-7.0.0 +) +FetchContent_GetProperties(msgpack) +if(NOT msgpack_POPULATED) + FetchContent_Populate(msgpack) +endif() +set(MSGPACK_INCLUDE ${msgpack_SOURCE_DIR}/include) + +# msgpack-c is used header-only with exceptions (no boost). Compiled with libc++ and +# -march=skylake so lmdblib's objects are ABI- and arch-compatible with the consumers +# (kvdb, wsdb) they are linked into. +add_compile_definitions(MSGPACK_NO_BOOST NO_PAR_ALGOS) +add_compile_options(-march=skylake -fconstexpr-steps=100000000 -fbracket-depth=1024) +add_compile_options(-Wno-deprecated-declarations -Wno-missing-field-initializers) +add_compile_options(-stdlib=libc++) + +include_directories( + ${CMAKE_CURRENT_SOURCE_DIR}/src + ${MSGPACK_INCLUDE} + ${LMDB_DIR} +) + +# --------------------------------------------------------------------------- +# liblmdblib.a — the C++ wrapper objects. +# --------------------------------------------------------------------------- +set(LMDBLIB_SOURCES + src/lmdblib/lmdb_cursor.cpp + src/lmdblib/lmdb_database.cpp + src/lmdblib/lmdb_db_transaction.cpp + src/lmdblib/lmdb_environment.cpp + src/lmdblib/lmdb_helpers.cpp + src/lmdblib/lmdb_read_transaction.cpp + src/lmdblib/lmdb_store.cpp + src/lmdblib/lmdb_store_base.cpp + src/lmdblib/lmdb_transaction.cpp + src/lmdblib/lmdb_write_transaction.cpp + src/lmdblib/queries.cpp +) +add_library(lmdblib STATIC ${LMDBLIB_SOURCES}) +add_dependencies(lmdblib lmdb) + +# --------------------------------------------------------------------------- +# Tests (vendored googletest; self-contained, no external dependencies). +# --------------------------------------------------------------------------- +option(LMDBLIB_BUILD_TESTS "Build lmdblib C++ tests" ON) +if(LMDBLIB_BUILD_TESTS) + add_link_options(-stdlib=libc++) + FetchContent_Declare( + googletest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG v1.13.0 + ) + set(INSTALL_GTEST OFF CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(googletest) + add_executable(lmdblib_tests + src/lmdblib/lmdb_store.test.cpp + src/lmdblib/lmdb_environment.test.cpp + ) + target_link_libraries(lmdblib_tests PRIVATE lmdblib lmdb GTest::gtest_main pthread) +endif() diff --git a/native-packages/lmdblib/cpp/src/lmdblib/fixtures.hpp b/native-packages/lmdblib/cpp/src/lmdblib/fixtures.hpp new file mode 100644 index 000000000000..6491bbead88a --- /dev/null +++ b/native-packages/lmdblib/cpp/src/lmdblib/fixtures.hpp @@ -0,0 +1,36 @@ +#include "lmdblib/types.hpp" +#include +#include +#include + +namespace azteclabs::lmdblib { +const uint32_t NUM_VALUES = 1024; + +inline std::string random_string() { + static thread_local std::mt19937 gen(std::random_device{}()); + std::stringstream ss; + ss << std::uniform_int_distribution{}(gen); + return ss.str(); +} + +inline std::string random_temp_directory() { + std::stringstream ss; + ss << "/tmp/lmdb/" << random_string(); + return ss.str(); +} + +inline std::vector serialise(std::string key) { + std::vector data(key.begin(), key.end()); + return data; +} + +inline Key get_key(int64_t keyCount) { + return serialise((std::stringstream() << "Key" << keyCount).str()); +} + +inline Value get_value(int64_t keyCount, int64_t valueCount) { + return serialise( + (std::stringstream() << "Key" << keyCount << "Data" << valueCount).str()); +} + +} // namespace azteclabs::lmdblib \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_cursor.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_cursor.cpp similarity index 90% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_cursor.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_cursor.cpp index 809d9bd0dffb..c895dca5aa25 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_cursor.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_cursor.cpp @@ -1,11 +1,11 @@ -#include "barretenberg/lmdblib/lmdb_cursor.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/lmdblib/lmdb_read_transaction.hpp" -#include "barretenberg/lmdblib/queries.hpp" +#include "lmdblib/lmdb_cursor.hpp" +#include "lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_read_transaction.hpp" +#include "lmdblib/queries.hpp" #include "lmdb.h" #include -namespace bb::lmdblib { +namespace azteclabs::lmdblib { LMDBCursor::LMDBCursor(LMDBReadTransaction::SharedPtr tx, LMDBDatabase::SharedPtr db, uint64_t id) : _tx(tx) , _db(db) @@ -99,4 +99,4 @@ bool LMDBCursor::count_until_prev(const Key& key, uint64_t& count) const return lmdb_queries::count_until_prev(*this, key, count); } -} // namespace bb::lmdblib +} // namespace azteclabs::lmdblib diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_cursor.hpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_cursor.hpp similarity index 93% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_cursor.hpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_cursor.hpp index e1a0a1d22b6a..dc7eb374a01e 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_cursor.hpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_cursor.hpp @@ -1,10 +1,10 @@ #pragma once -#include "barretenberg/lmdblib/types.hpp" +#include "lmdblib/types.hpp" #include "lmdb.h" #include #include -namespace bb::lmdblib { +namespace azteclabs::lmdblib { class LMDBReadTransaction; class LMDBDatabase; class LMDBCursor { @@ -41,4 +41,4 @@ class LMDBCursor { uint64_t _id; MDB_cursor* _cursor; }; -} // namespace bb::lmdblib +} // namespace azteclabs::lmdblib diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_database.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_database.cpp similarity index 75% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_database.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_database.cpp index 27c910aad7d8..2ef64ff17f78 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_database.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_database.cpp @@ -1,13 +1,13 @@ -#include "barretenberg/lmdblib/lmdb_database.hpp" -#include "barretenberg/lmdblib/lmdb_db_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/lmdblib/lmdb_read_transaction.hpp" -#include "barretenberg/lmdblib/types.hpp" +#include "lmdblib/lmdb_database.hpp" +#include "lmdblib/lmdb_db_transaction.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_read_transaction.hpp" +#include "lmdblib/types.hpp" #include "lmdb.h" #include -namespace bb::lmdblib { +namespace azteclabs::lmdblib { LMDBDatabase::LMDBDatabase(LMDBEnvironment::SharedPtr env, const LMDBDatabaseCreationTransaction& transaction, const std::string& name, @@ -59,7 +59,8 @@ DBStats LMDBDatabase::get_stats(LMDBReadTransaction& tx) { MDB_stat stat; call_lmdb_func(mdb_stat, tx.underlying(), underlying(), &stat); - return DBStats(name(), stat); + uint64_t totalUsedSize = stat.ms_psize * (stat.ms_branch_pages + stat.ms_leaf_pages + stat.ms_overflow_pages); + return DBStats(name(), stat.ms_entries, totalUsedSize); } -} // namespace bb::lmdblib +} // namespace azteclabs::lmdblib diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_database.hpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_database.hpp similarity index 89% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_database.hpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_database.hpp index 2f8960647584..9fa353be386a 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_database.hpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_database.hpp @@ -1,8 +1,8 @@ #pragma once -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/types.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/types.hpp" -namespace bb::lmdblib { +namespace azteclabs::lmdblib { class LMDBDatabaseCreationTransaction; class LMDBReadTransaction; @@ -41,4 +41,4 @@ class LMDBDatabase { MDB_dbi _dbi; LMDBEnvironment::SharedPtr environment; }; -} // namespace bb::lmdblib +} // namespace azteclabs::lmdblib diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_db_transaction.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_db_transaction.cpp similarity index 80% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_db_transaction.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_db_transaction.cpp index 549e3c78baad..c829f8421988 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_db_transaction.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_db_transaction.cpp @@ -1,9 +1,9 @@ -#include "barretenberg/lmdblib/lmdb_db_transaction.hpp" +#include "lmdblib/lmdb_db_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_helpers.hpp" #include -namespace bb::lmdblib { +namespace azteclabs::lmdblib { LMDBDatabaseCreationTransaction::LMDBDatabaseCreationTransaction(LMDBEnvironment::SharedPtr env) : LMDBTransaction(std::move(env)) {} @@ -25,4 +25,4 @@ void LMDBDatabaseCreationTransaction::try_abort() { LMDBTransaction::abort(); } -} // namespace bb::lmdblib \ No newline at end of file +} // namespace azteclabs::lmdblib \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_db_transaction.hpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_db_transaction.hpp similarity index 88% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_db_transaction.hpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_db_transaction.hpp index e0fd2a3ad25c..6d7e8f13c66a 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_db_transaction.hpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_db_transaction.hpp @@ -1,7 +1,7 @@ #pragma once -#include "barretenberg/lmdblib/lmdb_transaction.hpp" +#include "lmdblib/lmdb_transaction.hpp" -namespace bb::lmdblib { +namespace azteclabs::lmdblib { /* * RAII wrapper to construct a transaction for the purpose of creating/opening a database @@ -21,4 +21,4 @@ class LMDBDatabaseCreationTransaction : public LMDBTransaction { void try_abort(); }; -} // namespace bb::lmdblib \ No newline at end of file +} // namespace azteclabs::lmdblib \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_environment.cpp similarity index 93% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_environment.cpp index ab712c7fa2b5..50bcbabf4749 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_environment.cpp @@ -1,12 +1,12 @@ -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_helpers.hpp" #include "lmdb.h" #include #include #include #include -namespace bb::lmdblib { +namespace azteclabs::lmdblib { LMDBEnvironment::LMDBEnvironment( const std::string& directory, uint64_t mapSizeKB, uint32_t maxNumDBs, uint32_t maxNumReaders, bool ephemeral) @@ -83,4 +83,4 @@ uint64_t LMDBEnvironment::get_data_file_size() const } return 0; } -} // namespace bb::lmdblib +} // namespace azteclabs::lmdblib diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.hpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_environment.hpp similarity index 98% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.hpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_environment.hpp index c463ca74675c..8b42fb43d098 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.hpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_environment.hpp @@ -7,7 +7,7 @@ #include #include #include -namespace bb::lmdblib { +namespace azteclabs::lmdblib { /* * RAII wrapper around an LMDB environment. @@ -96,4 +96,4 @@ class LMDBEnvironment { ResourceGuard _readGuard; ResourceGuard _writeGuard; }; -} // namespace bb::lmdblib +} // namespace azteclabs::lmdblib diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.test.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_environment.test.cpp similarity index 92% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.test.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_environment.test.cpp index f196a20765d3..e1659873b7f0 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.test.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_environment.test.cpp @@ -11,18 +11,15 @@ #include #include -#include "barretenberg/common/serialize.hpp" -#include "barretenberg/common/streams.hpp" -#include "barretenberg/common/test.hpp" -#include "barretenberg/lmdblib/fixtures.hpp" -#include "barretenberg/lmdblib/lmdb_database.hpp" -#include "barretenberg/lmdblib/lmdb_db_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_read_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_write_transaction.hpp" -#include "barretenberg/lmdblib/queries.hpp" - -using namespace bb::lmdblib; +#include "lmdblib/fixtures.hpp" +#include "lmdblib/lmdb_database.hpp" +#include "lmdblib/lmdb_db_transaction.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_read_transaction.hpp" +#include "lmdblib/lmdb_write_transaction.hpp" +#include "lmdblib/queries.hpp" + +using namespace azteclabs::lmdblib; class LMDBEnvironmentTest : public testing::Test { protected: diff --git a/native-packages/lmdblib/cpp/src/lmdblib/lmdb_helpers.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_helpers.cpp new file mode 100644 index 000000000000..ee79167ead45 --- /dev/null +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_helpers.cpp @@ -0,0 +1,61 @@ +#include "lmdblib/lmdb_helpers.hpp" +#include "lmdb.h" +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#include +#define htole64(x) OSSwapHostToLittleInt64(x) +#define le64toh(x) OSSwapLittleToHostInt64(x) +#else +#include +#endif + +namespace azteclabs::lmdblib { +void throw_error(const std::string &errorString, int error) { + std::stringstream ss; + ss << errorString << ": " << error << " - " << mdb_strerror(error) + << std::endl; + throw std::runtime_error(ss.str()); +} + +std::vector serialise_key(uint8_t key) { return {key}; } + +void deserialise_key(void *data, uint8_t &key) { + uint8_t *p = static_cast(data); + key = *p; +} + +// 64 bit integers are stored in little endian byte order +std::vector serialise_key(uint64_t key) { + uint64_t le = htole64(key); + const uint8_t *p = reinterpret_cast(&le); + return std::vector(p, p + sizeof(key)); +} + +void deserialise_key(void *data, uint64_t &key) { + uint64_t le = 0; + std::memcpy(&le, data, sizeof(le)); + key = le64toh(le); +} + +int size_cmp(const MDB_val *a, const MDB_val *b) { + if (a->mv_size < b->mv_size) { + return -1; + } + return (a->mv_size > b->mv_size) ? 1 : 0; +} + +std::vector mdb_val_to_vector(const MDB_val &dbVal) { + const uint8_t *p = static_cast(dbVal.mv_data); + return std::vector(p, p + dbVal.mv_size); +} + +void copy_to_vector(const MDB_val &dbVal, std::vector &target) { + std::vector temp = mdb_val_to_vector(dbVal); + target.swap(temp); +} +} // namespace azteclabs::lmdblib \ No newline at end of file diff --git a/native-packages/lmdblib/cpp/src/lmdblib/lmdb_helpers.hpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_helpers.hpp new file mode 100644 index 000000000000..66757e404ad6 --- /dev/null +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_helpers.hpp @@ -0,0 +1,83 @@ + +#pragma once +#include "lmdb.h" +#include +#include +#include +#include +namespace azteclabs::lmdblib { +// Concatenate args into a string (replaces barretenberg's common `format` +// helper). +template std::string format(Args &&...args) { + std::ostringstream os; + (os << ... << args); + return os.str(); +} + +void throw_error(const std::string &errorString, int error); + +int size_cmp(const MDB_val *a, const MDB_val *b); + +std::vector serialise_key(uint8_t key); +std::vector serialise_key(uint64_t key); + +void deserialise_key(void *data, uint8_t &key); +void deserialise_key(void *data, uint64_t &key); + +// Fixed-width "field-like" keys (e.g. a 32-byte field element): serialised as +// the type's raw little-endian bytes. Templated so lmdblib needs no concrete +// big-int type from barretenberg; callers (e.g. the merkle trees) instantiate +// it with their own key type. The byte layout is unchanged from the old uint256 +// overload. +template std::vector serialise_key(const T &key) { + std::vector buf(sizeof(T)); + std::memcpy(buf.data(), &key, sizeof(T)); + return buf; +} + +template void deserialise_key(void *data, T &key) { + std::memcpy(&key, data, sizeof(T)); +} + +template int value_cmp(const MDB_val *a, const MDB_val *b) { + T lhs; + T rhs; + deserialise_key(a->mv_data, lhs); + deserialise_key(b->mv_data, rhs); + if (lhs < rhs) { + return -1; + } + if (lhs > rhs) { + return 1; + } + return 0; +} + +std::vector mdb_val_to_vector(const MDB_val &dbVal); +void copy_to_vector(const MDB_val &dbVal, std::vector &target); + +template +bool call_lmdb_func(int (*f)(TArgs...), TArgs... args) { + int error = f(args...); + return error == 0; +} + +template +int call_lmdb_func_with_return(int (*f)(TArgs...), TArgs... args) { + return f(args...); +} + +template +void call_lmdb_func(const std::string &errorString, int (*f)(TArgs...), + TArgs... args) { + int error = f(args...); + if (error != 0) { + throw_error(errorString, error); + } +} + +template +void call_lmdb_func(void (*f)(TArgs...), TArgs... args) { + f(args...); +} +} // namespace azteclabs::lmdblib \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_read_transaction.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_read_transaction.cpp similarity index 53% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_read_transaction.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_read_transaction.cpp index 51a8c7f754cd..2c54ac557de8 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_read_transaction.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_read_transaction.cpp @@ -1,9 +1,9 @@ -#include "barretenberg/lmdblib/lmdb_read_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_read_transaction.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_helpers.hpp" #include -namespace bb::lmdblib { +namespace azteclabs::lmdblib { LMDBReadTransaction::LMDBReadTransaction(LMDBEnvironment::SharedPtr env) : LMDBTransaction(env, true) {} @@ -13,4 +13,4 @@ LMDBReadTransaction::~LMDBReadTransaction() LMDBTransaction::abort(); _environment->release_reader(); } -} // namespace bb::lmdblib +} // namespace azteclabs::lmdblib diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_read_transaction.hpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_read_transaction.hpp similarity index 71% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_read_transaction.hpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_read_transaction.hpp index 298651d10792..b3c0f977c914 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_read_transaction.hpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_read_transaction.hpp @@ -1,10 +1,8 @@ #pragma once -#include "barretenberg/common/serialize.hpp" -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/lmdblib/lmdb_transaction.hpp" -#include "barretenberg/lmdblib/queries.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_transaction.hpp" +#include "lmdblib/queries.hpp" #include #include #include @@ -12,7 +10,7 @@ #include #include -namespace bb::lmdblib { +namespace azteclabs::lmdblib { /** * RAII wrapper around a read transaction. @@ -32,4 +30,4 @@ class LMDBReadTransaction : public LMDBTransaction { ~LMDBReadTransaction() override; }; -} // namespace bb::lmdblib \ No newline at end of file +} // namespace azteclabs::lmdblib \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_store.cpp similarity index 95% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_store.cpp index 15165a93d910..61d574ab486d 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_store.cpp @@ -1,9 +1,9 @@ -#include "barretenberg/lmdblib/lmdb_store.hpp" -#include "barretenberg/lmdblib/lmdb_database.hpp" -#include "barretenberg/lmdblib/lmdb_db_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_store_base.hpp" -#include "barretenberg/lmdblib/lmdb_write_transaction.hpp" -#include "barretenberg/lmdblib/types.hpp" +#include "lmdblib/lmdb_store.hpp" +#include "lmdblib/lmdb_database.hpp" +#include "lmdblib/lmdb_db_transaction.hpp" +#include "lmdblib/lmdb_store_base.hpp" +#include "lmdblib/lmdb_write_transaction.hpp" +#include "lmdblib/types.hpp" #include "lmdb.h" #include #include @@ -13,7 +13,7 @@ #include #include -namespace bb::lmdblib { +namespace azteclabs::lmdblib { LMDBStore::LMDBStore(std::string directory, uint64_t mapSizeKb, uint64_t maxNumReaders, uint64_t maxDbs, bool ephemeral) : LMDBStoreBase(std::move(directory), mapSizeKb, maxNumReaders, maxDbs, ephemeral) {} @@ -228,4 +228,4 @@ LMDBStore::Cursor::Ptr LMDBStore::create_cursor(ReadTransaction::SharedPtr tx, c Database::SharedPtr db = get_database(dbName); return std::make_unique(tx, db, _environment->getNextId()); } -} // namespace bb::lmdblib +} // namespace azteclabs::lmdblib diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store.hpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_store.hpp similarity index 84% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store.hpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_store.hpp index 299dc9f65c12..d55d1d5bd40f 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store.hpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_store.hpp @@ -1,20 +1,20 @@ #pragma once -#include "barretenberg/lmdblib/lmdb_cursor.hpp" -#include "barretenberg/lmdblib/lmdb_database.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_read_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_store_base.hpp" -#include "barretenberg/lmdblib/lmdb_write_transaction.hpp" -#include "barretenberg/lmdblib/queries.hpp" -#include "barretenberg/lmdblib/types.hpp" +#include "lmdblib/lmdb_cursor.hpp" +#include "lmdblib/lmdb_database.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_read_transaction.hpp" +#include "lmdblib/lmdb_store_base.hpp" +#include "lmdblib/lmdb_write_transaction.hpp" +#include "lmdblib/queries.hpp" +#include "lmdblib/types.hpp" #include #include #include #include #include #include -namespace bb::lmdblib { +namespace azteclabs::lmdblib { /** * Implements a basic lmdb store. Consisting of an environment, any number of databases, transactions and cursors @@ -73,4 +73,4 @@ class LMDBStore : public LMDBStoreBase { // Throws if any of the databases are not found std::vector get_databases(const std::vector& puts) const; }; -} // namespace bb::lmdblib +} // namespace azteclabs::lmdblib diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store.test.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_store.test.cpp similarity index 98% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store.test.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_store.test.cpp index 75c677adc14b..e297e6c36818 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store.test.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_store.test.cpp @@ -12,21 +12,18 @@ #include #include -#include "barretenberg/common/serialize.hpp" -#include "barretenberg/common/streams.hpp" -#include "barretenberg/common/test.hpp" -#include "barretenberg/lmdblib/fixtures.hpp" -#include "barretenberg/lmdblib/lmdb_cursor.hpp" -#include "barretenberg/lmdblib/lmdb_database.hpp" -#include "barretenberg/lmdblib/lmdb_db_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_read_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_store.hpp" -#include "barretenberg/lmdblib/lmdb_write_transaction.hpp" -#include "barretenberg/lmdblib/queries.hpp" -#include "barretenberg/lmdblib/types.hpp" - -using namespace bb::lmdblib; +#include "lmdblib/fixtures.hpp" +#include "lmdblib/lmdb_cursor.hpp" +#include "lmdblib/lmdb_database.hpp" +#include "lmdblib/lmdb_db_transaction.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_read_transaction.hpp" +#include "lmdblib/lmdb_store.hpp" +#include "lmdblib/lmdb_write_transaction.hpp" +#include "lmdblib/queries.hpp" +#include "lmdblib/types.hpp" + +using namespace azteclabs::lmdblib; class LMDBStoreTest : public testing::Test { protected: diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store_base.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_store_base.cpp similarity index 94% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store_base.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_store_base.cpp index 459510da23bb..efc85c1de911 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store_base.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_store_base.cpp @@ -1,6 +1,6 @@ -#include "barretenberg/lmdblib/lmdb_store_base.hpp" +#include "lmdblib/lmdb_store_base.hpp" -namespace bb::lmdblib { +namespace azteclabs::lmdblib { LMDBStoreBase::LMDBStoreBase( std::string directory, uint64_t mapSizeKb, uint64_t maxNumReaders, uint64_t maxDbs, bool ephemeral) : _dbDirectory(std::move(directory)) @@ -44,4 +44,4 @@ void LMDBStoreBase::copy_store(const std::string& dstPath, bool compact) static_cast(compact ? MDB_CP_COMPACT : 0)); } -} // namespace bb::lmdblib \ No newline at end of file +} // namespace azteclabs::lmdblib \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store_base.hpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_store_base.hpp similarity index 79% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store_base.hpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_store_base.hpp index fd03e29c4ea4..234721b6db2d 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store_base.hpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_store_base.hpp @@ -1,11 +1,11 @@ #pragma once -#include "barretenberg/lmdblib/lmdb_db_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_read_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_write_transaction.hpp" +#include "lmdblib/lmdb_db_transaction.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_read_transaction.hpp" +#include "lmdblib/lmdb_write_transaction.hpp" -namespace bb::lmdblib { +namespace azteclabs::lmdblib { class LMDBStoreBase { public: using ReadTransaction = LMDBReadTransaction; @@ -28,4 +28,4 @@ class LMDBStoreBase { std::string _dbDirectory; LMDBEnvironment::SharedPtr _environment; }; -} // namespace bb::lmdblib +} // namespace azteclabs::lmdblib diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_transaction.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_transaction.cpp similarity index 84% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_transaction.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_transaction.cpp index 45b1c9d1c2d2..45741465ed78 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_transaction.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_transaction.cpp @@ -1,10 +1,10 @@ -#include "barretenberg/lmdblib/lmdb_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_transaction.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_helpers.hpp" #include #include -namespace bb::lmdblib { +namespace azteclabs::lmdblib { LMDBTransaction::LMDBTransaction(std::shared_ptr env, bool readOnly) : _environment(std::move(env)) , _id(_environment->getNextId()) @@ -45,4 +45,4 @@ bool LMDBTransaction::get_value(std::vector& key, uint64_t& data, const { return lmdb_queries::get_value(key, data, db, *this); } -} // namespace bb::lmdblib \ No newline at end of file +} // namespace azteclabs::lmdblib \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_transaction.hpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_transaction.hpp similarity index 96% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_transaction.hpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_transaction.hpp index d5bc5c4b0198..d8d55ae7e25f 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_transaction.hpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_transaction.hpp @@ -1,12 +1,12 @@ #pragma once -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/queries.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/queries.hpp" #include "lmdb.h" #include #include #include -namespace bb::lmdblib { +namespace azteclabs::lmdblib { /* * Abstract base class to represent and LMDB transaction. @@ -125,4 +125,4 @@ void LMDBTransaction::get_all_values_lesser_or_equal_key(const T& key, { lmdb_queries::get_all_values_lesser_or_equal_key(key, data, db, *this); } -} // namespace bb::lmdblib +} // namespace azteclabs::lmdblib diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_write_transaction.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_write_transaction.cpp similarity index 80% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_write_transaction.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_write_transaction.cpp index b4d3151ce03e..a1f138413288 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_write_transaction.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_write_transaction.cpp @@ -1,15 +1,15 @@ -#include "barretenberg/lmdblib/lmdb_write_transaction.hpp" +#include "lmdblib/lmdb_write_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_database.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/lmdblib/queries.hpp" +#include "lmdblib/lmdb_database.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_helpers.hpp" +#include "lmdblib/queries.hpp" #include "lmdb.h" #include -namespace bb::lmdblib { +namespace azteclabs::lmdblib { LMDBWriteTransaction::LMDBWriteTransaction(LMDBEnvironment::SharedPtr env) : LMDBTransaction(std::move(env)) @@ -54,4 +54,4 @@ void LMDBWriteTransaction::delete_value(Key& key, Value& value, const LMDBDataba { lmdb_queries::delete_value(key, value, db, *this); } -} // namespace bb::lmdblib +} // namespace azteclabs::lmdblib diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_write_transaction.hpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_write_transaction.hpp similarity index 88% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_write_transaction.hpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_write_transaction.hpp index 71674409b6fc..31410602fd27 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_write_transaction.hpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_write_transaction.hpp @@ -1,17 +1,15 @@ #pragma once -#include "barretenberg/common/serialize.hpp" -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/lmdblib/lmdb_database.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/lmdblib/lmdb_transaction.hpp" -#include "barretenberg/lmdblib/queries.hpp" +#include "lmdblib/lmdb_database.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_transaction.hpp" +#include "lmdblib/queries.hpp" #include "lmdb.h" #include #include #include -namespace bb::lmdblib { +namespace azteclabs::lmdblib { /** * RAII wrapper for an LMDB write transaction. @@ -91,4 +89,4 @@ void LMDBWriteTransaction::delete_all_values_lesser_or_equal_key(const T& key, c { lmdb_queries::delete_all_values_lesser_or_equal_key(key, db, *this); } -} // namespace bb::lmdblib \ No newline at end of file +} // namespace azteclabs::lmdblib \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/queries.cpp b/native-packages/lmdblib/cpp/src/lmdblib/queries.cpp similarity index 93% rename from barretenberg/cpp/src/barretenberg/lmdblib/queries.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/queries.cpp index 9bbb54ee8cb4..3d4889a0e5b4 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/queries.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/queries.cpp @@ -1,16 +1,16 @@ -#include "barretenberg/lmdblib/queries.hpp" -#include "barretenberg/lmdblib/lmdb_cursor.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/lmdblib/lmdb_write_transaction.hpp" -#include "barretenberg/lmdblib/types.hpp" +#include "lmdblib/queries.hpp" +#include "lmdblib/lmdb_cursor.hpp" +#include "lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_write_transaction.hpp" +#include "lmdblib/types.hpp" #include "lmdb.h" #include #include -namespace bb::lmdblib::lmdb_queries { +namespace azteclabs::lmdblib::lmdb_queries { void put_value( - Key& key, Value& data, const LMDBDatabase& db, bb::lmdblib::LMDBWriteTransaction& tx, bool duplicatesPermitted) + Key& key, Value& data, const LMDBDatabase& db, azteclabs::lmdblib::LMDBWriteTransaction& tx, bool duplicatesPermitted) { MDB_val dbKey; dbKey.mv_size = key.size(); @@ -36,7 +36,7 @@ void put_value( void put_value(Key& key, const uint64_t& data, const LMDBDatabase& db, - bb::lmdblib::LMDBWriteTransaction& tx, + azteclabs::lmdblib::LMDBWriteTransaction& tx, bool duplicatesPermitted) { MDB_val dbKey; @@ -56,7 +56,7 @@ void put_value(Key& key, call_lmdb_func("mdb_put", mdb_put, tx.underlying(), db.underlying(), &dbKey, &dbVal, flags); } -void delete_value(Key& key, const LMDBDatabase& db, bb::lmdblib::LMDBWriteTransaction& tx) +void delete_value(Key& key, const LMDBDatabase& db, azteclabs::lmdblib::LMDBWriteTransaction& tx) { MDB_val dbKey; dbKey.mv_size = key.size(); @@ -69,7 +69,7 @@ void delete_value(Key& key, const LMDBDatabase& db, bb::lmdblib::LMDBWriteTransa } } -void delete_value(Key& key, Value& value, const LMDBDatabase& db, bb::lmdblib::LMDBWriteTransaction& tx) +void delete_value(Key& key, Value& value, const LMDBDatabase& db, azteclabs::lmdblib::LMDBWriteTransaction& tx) { MDB_val dbKey; dbKey.mv_size = key.size(); @@ -85,7 +85,7 @@ void delete_value(Key& key, Value& value, const LMDBDatabase& db, bb::lmdblib::L } } -bool get_value(Key& key, Value& data, const LMDBDatabase& db, const bb::lmdblib::LMDBTransaction& tx) +bool get_value(Key& key, Value& data, const LMDBDatabase& db, const azteclabs::lmdblib::LMDBTransaction& tx) { MDB_val dbKey; dbKey.mv_size = key.size(); @@ -99,7 +99,7 @@ bool get_value(Key& key, Value& data, const LMDBDatabase& db, const bb::lmdblib: return true; } -bool get_value(Key& key, uint64_t& data, const LMDBDatabase& db, const bb::lmdblib::LMDBTransaction& tx) +bool get_value(Key& key, uint64_t& data, const LMDBDatabase& db, const azteclabs::lmdblib::LMDBTransaction& tx) { MDB_val dbKey; dbKey.mv_size = key.size(); @@ -311,4 +311,4 @@ bool count_until_prev_dup(const LMDBCursor& cursor, const Key& key, uint64_t& co { return count_until_next_dup(cursor, key, count, MDB_PREV_NODUP); } -} // namespace bb::lmdblib::lmdb_queries +} // namespace azteclabs::lmdblib::lmdb_queries diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/queries.hpp b/native-packages/lmdblib/cpp/src/lmdblib/queries.hpp similarity index 98% rename from barretenberg/cpp/src/barretenberg/lmdblib/queries.hpp rename to native-packages/lmdblib/cpp/src/lmdblib/queries.hpp index 9f5ac1d09f2b..c71fb2a907e5 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/queries.hpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/queries.hpp @@ -1,14 +1,14 @@ #pragma once -#include "barretenberg/lmdblib/lmdb_database.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/lmdblib/types.hpp" +#include "lmdblib/lmdb_database.hpp" +#include "lmdblib/lmdb_helpers.hpp" +#include "lmdblib/types.hpp" #include "lmdb.h" #include #include #include -namespace bb::lmdblib { +namespace azteclabs::lmdblib { class LMDBTransaction; class LMDBWriteTransaction; @@ -468,4 +468,4 @@ bool count_until_next_dup(const LMDBCursor& cursor, const Key& key, uint64_t& co bool count_until_prev_dup(const LMDBCursor& cursor, const Key& key, uint64_t& count); } // namespace lmdb_queries -} // namespace bb::lmdblib +} // namespace azteclabs::lmdblib diff --git a/native-packages/lmdblib/cpp/src/lmdblib/types.hpp b/native-packages/lmdblib/cpp/src/lmdblib/types.hpp new file mode 100644 index 000000000000..fe8555369f3a --- /dev/null +++ b/native-packages/lmdblib/cpp/src/lmdblib/types.hpp @@ -0,0 +1,64 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +namespace azteclabs::lmdblib { +using Key = std::vector; +using Value = std::vector; +using KeysVector = std::vector; +using ValuesVector = std::vector; +using KeyValuesPair = std::pair; +using OptionalValues = std::optional; +using OptionalValuesVector = std::vector; +using KeyDupValuesVector = std::vector; +using KeyOptionalValuesPair = std::pair; +using KeyOptionalValuesVector = std::vector; + +/** + * @brief Storage statistics for a single named lmdb database. + * + * Plain wire value type owned by lmdblib (it is what lmdb's get_stats reports). + */ +struct DBStats { + std::string name; + uint64_t numDataItems; + uint64_t totalUsedSize; + + DBStats() = default; + DBStats(const DBStats &other) = default; + DBStats(DBStats &&other) noexcept { *this = std::move(other); } + ~DBStats() = default; + DBStats(const std::string &name, uint64_t numDataItems, + uint64_t totalUsedSize) + : name(name), numDataItems(numDataItems), totalUsedSize(totalUsedSize) {} + + MSGPACK_DEFINE_MAP(name, numDataItems, totalUsedSize) + + bool operator==(const DBStats &other) const { + return name == other.name && numDataItems == other.numDataItems && + totalUsedSize == other.totalUsedSize; + } + + DBStats &operator=(const DBStats &other) = default; + + DBStats &operator=(DBStats &&other) noexcept { + if (this != &other) { + name = std::move(other.name); + numDataItems = other.numDataItems; + totalUsedSize = other.totalUsedSize; + } + return *this; + } + + friend std::ostream &operator<<(std::ostream &os, const DBStats &stats) { + os << "DB " << stats.name << ", num items: " << stats.numDataItems + << ", total used size: " << stats.totalUsedSize; + return os; + } +}; + +} // namespace azteclabs::lmdblib \ No newline at end of file diff --git a/native-packages/wsdb/.gitignore b/native-packages/wsdb/.gitignore new file mode 100644 index 000000000000..f80c50825486 --- /dev/null +++ b/native-packages/wsdb/.gitignore @@ -0,0 +1,12 @@ +ts/ +cpp/build/ +# ipc-codegen emits the server dispatch into a generated/ subdir at build time. +**/generated/ +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/sdks +!.yarn/versions +node_modules +*.log diff --git a/native-packages/wsdb/.rebuild_patterns b/native-packages/wsdb/.rebuild_patterns new file mode 100644 index 000000000000..6c2d11b6e8ed --- /dev/null +++ b/native-packages/wsdb/.rebuild_patterns @@ -0,0 +1,8 @@ +^native-packages/wsdb/bootstrap\.sh$ +^native-packages/wsdb/package\.json$ +^native-packages/wsdb/\.rebuild_patterns$ +^native-packages/wsdb/wsdb_schema\.jsonc$ +^native-packages/wsdb/cpp/ +^native-packages/lmdblib/ +^ipc-codegen/src/.*\.ts$ +^ipc-codegen/templates/ diff --git a/native-packages/wsdb/.yarnrc.yml b/native-packages/wsdb/.yarnrc.yml new file mode 100644 index 000000000000..3186f3f0795a --- /dev/null +++ b/native-packages/wsdb/.yarnrc.yml @@ -0,0 +1 @@ +nodeLinker: node-modules diff --git a/native-packages/wsdb/bootstrap.sh b/native-packages/wsdb/bootstrap.sh new file mode 100755 index 000000000000..3d78adb7b50e --- /dev/null +++ b/native-packages/wsdb/bootstrap.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +source $(git rev-parse --show-toplevel)/ci3/source_bootstrap + +ROOT=$(git rev-parse --show-toplevel) +PKG="$ROOT/native-packages/wsdb" +WSDB_BINARY=aztec-wsdb +# This package owns the wire contract; both the TS client and the C++ server +# codegen from this schema. +WSDB_SCHEMA="$PKG/wsdb_schema.jsonc" + +hash=$(hash_str \ + $(../../barretenberg/cpp/bootstrap.sh hash) \ + $(../../ipc-runtime/bootstrap.sh hash) \ + $(../lmdblib/bootstrap.sh hash) \ + $(cache_content_hash .rebuild_patterns)) + +function generate_ts_package { + node --experimental-strip-types --experimental-transform-types --no-warnings \ + "$ROOT/ipc-codegen/src/generate.ts" \ + --schema "$WSDB_SCHEMA" \ + --lang ts \ + --client \ + --out "$PKG/ts/src/generated" \ + --package "$PKG/ts" \ + --package-name @aztec/wsdb \ + --binary-name "$WSDB_BINARY" \ + --package-transports uds,shm \ + --package-ipc-path-args 'msgpack,run,--input,{path}' +} + +# Build the standalone aztec-wsdb binary against the prebuilt barretenberg +# artifacts (default preset -> barretenberg/cpp/build). bb is never rebuilt here. +function build_native { + local build_dir="cpp/build" + CC=$(which clang) CXX=$(which clang++) cmake -S cpp -B "$build_dir" -G Ninja >/dev/null + cmake --build "$build_dir" --target "$WSDB_BINARY" wsdb_tests + local target_dir="ts/build/$(arch)-$(os)" + mkdir -p "$target_dir" + cp "$build_dir/bin/$WSDB_BINARY" "$target_dir/$WSDB_BINARY" +} + +function build { + echo_header "wsdb build" + generate_ts_package + build_native + npm_install_deps + yarn build + (cd ts && ./scripts/prepare_arch_packages.sh "$(arch)-$(os)=build/$(arch)-$(os)/$WSDB_BINARY") +} + +# Emit test commands for the CI test engine. The decoupled wsdb_tests use no +# barretenberg headers; they link libbarretenberg.a only for the poseidon2 c_bind. +# (The optional bb-header parity/equivalence target, WSDB_BUILD_BB_TESTS, is manual.) +function test_cmds { + echo "$hash:CPUS=8:TIMEOUT=600s native-packages/wsdb/cpp/build/bin/wsdb_tests" +} + +# Manual: build then run the tests directly. +function test { + echo_header "wsdb test" + build + test_cmds | filter_test_cmds | parallelize +} + +function clean { + rm -rf ts node_modules cpp/build +} + +function release { + generate_ts_package + build_native + npm_install_deps + yarn build + (cd ts && ./scripts/prepare_arch_packages.sh) + for package_dir in ts/packages/*; do + (cd "$package_dir" && retry "deploy_npm ${REF_NAME#v}") + done + (cd ts && retry "deploy_npm ${REF_NAME#v}") +} + +export -f generate_ts_package build_native build test_cmds test clean release + +case "$cmd" in + "") + build + ;; + "hash") + echo "$hash" + ;; + *) + default_cmd_handler "$@" + ;; +esac diff --git a/native-packages/wsdb/cpp/CMakeLists.txt b/native-packages/wsdb/cpp/CMakeLists.txt new file mode 100644 index 000000000000..40f5efb192d7 --- /dev/null +++ b/native-packages/wsdb/cpp/CMakeLists.txt @@ -0,0 +1,268 @@ +# Standalone build for the aztec-wsdb world-state database service. +# +# This package owns the wsdb service: the world_state engine, the persistent +# content-addressed merkle storage, the lmdb tree store, and the IPC server. +# It depends on barretenberg only as a *prebuilt* artifact: bb is assumed to be +# already built in place (default preset -> barretenberg/cpp/build), and this +# project links the resulting static archives. It never (re)builds bb. +# +# Link target: the full libbarretenberg.a (for the generic merkle core), plus the +# lmdblib sibling package's liblmdblib.a (the lmdb C++ wrapper, which barretenberg +# no longer hosts). bb-external is intentionally NOT usable here -- it excludes +# crypto_merkle_tree_objects (see barretenberg/cpp/src/CMakeLists.txt). + +cmake_minimum_required(VERSION 3.24) +# C is enabled for any C sources pulled in transitively. +project(aztec_wsdb CXX C) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + +# Match barretenberg's prebuilt archive build (Release -> -O3 -DNDEBUG). bb headers +# (inline/template crypto + field code) are compiled into BOTH bb's Release archive +# and this build; compiling them here without NDEBUG/-O3 is an ODR violation that can +# silently diverge inlined results (e.g. tree leaf hashes). Default to Release. +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif() + +# --------------------------------------------------------------------------- +# Locate the prebuilt barretenberg tree. +# --------------------------------------------------------------------------- +get_filename_component(REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../.." ABSOLUTE) +set(BB_SRC "${REPO_ROOT}/barretenberg/cpp/src") +set(BB_BUILD "${REPO_ROOT}/barretenberg/cpp/build" CACHE PATH "barretenberg build dir") +set(IPC_RUNTIME_SRC "${REPO_ROOT}/ipc-runtime/cpp") +set(IPC_CODEGEN_DIR "${REPO_ROOT}/ipc-codegen") + +set(BB_LIB "${BB_BUILD}/lib/libbarretenberg.a") +# libbarretenberg.a deliberately excludes the env module (logstr / throw_or_abort_impl / +# hardware_concurrency) so consumers supply their own runtime. As a native command we +# link bb's env implementation, matching what the old in-bb aztec-wsdb target did. +set(ENV_LIB "${BB_BUILD}/lib/libenv.a") +set(IPC_RUNTIME_LIB "${BB_BUILD}/lib/libipc_runtime.a") + +# lmdblib sibling package: headers + the prebuilt wrapper archive (the lmdb C++ +# wrapper that barretenberg no longer hosts). +set(LMDBLIB_PKG "${REPO_ROOT}/native-packages/lmdblib") +set(LMDBLIB_SRC "${LMDBLIB_PKG}/cpp/src") +set(LMDBLIB_LIB "${LMDBLIB_PKG}/cpp/build/lib/liblmdblib.a") + +# wsdb consumes prebuilt static archives: barretenberg's (for the poseidon2 c_bind, +# under ${BB_BUILD}/lib) and the lmdblib sibling package's. lmdblib owns the LMDB and +# msgpack-c dependencies; wsdb gets them transitively (liblmdblib.a is built against +# them), so reference lmdblib's built artifacts rather than fetching our own copies. +set(DEPS_PREFIX "${CMAKE_BINARY_DIR}/_deps") +set(LMDB_INCLUDE "${LMDBLIB_PKG}/cpp/build/_deps/lmdb-src/libraries/liblmdb") +set(LMDB_LIB "${LMDBLIB_PKG}/cpp/build/lib/liblmdb.a") +set(MSGPACK_INCLUDE "${LMDBLIB_PKG}/cpp/build/_deps/msgpack-src/include") + +foreach(artifact "${BB_LIB}" "${ENV_LIB}" "${IPC_RUNTIME_LIB}" "${LMDBLIB_LIB}" "${LMDB_LIB}") + if(NOT EXISTS "${artifact}") + message(FATAL_ERROR "Missing prebuilt artifact: ${artifact}\n" + "Build barretenberg (cd barretenberg/cpp && cmake --preset default && cd build && ninja) " + "and lmdblib (cd native-packages/lmdblib && ./bootstrap.sh) first.") + endif() +endforeach() + +# Match the ABI-affecting compile flags of the bb build. bb is compiled with +# clang and -march=skylake; mixing toolchains/arch against bb's prebuilt static +# archives risks subtle ODR/template-instantiation mismatches, so match it. +add_compile_definitions(MSGPACK_NO_BOOST MSGPACK_USE_STD_VARIANT_ADAPTOR NO_PAR_ALGOS) +add_compile_options(-march=skylake -fconstexpr-steps=100000000 -fbracket-depth=1024) +add_compile_options(-Wno-deprecated-declarations -Wno-missing-field-initializers) +# bb's archives are built with libc++ (std::__1:: symbols). The system clang here +# links libstdc++ by default, leaving those symbols undefined, so force libc++ for +# both compile and link to match bb's ABI. +add_compile_options(-stdlib=libc++) +# Statically link libc++/libc++abi/libgcc so aztec-wsdb is self-contained (no +# libc++.so.1 runtime dependency), matching bb's own aztec-wsdb. Otherwise the +# binary fails to launch (exit 127) in minimal environments — e.g. a scaffolded +# user project — that don't ship the libc++ runtime. +add_link_options(-stdlib=libc++ -static-libstdc++ -static-libgcc) + +# The decoupled wsdb core/tests include NO barretenberg headers — only wsdb's own src, +# the bb-free sibling packages (lmdblib, ipc-runtime), msgpack-c and lmdb. bb is still +# *linked* (libbarretenberg.a) purely for the poseidon2 c_bind. BB_SRC is added back only +# on the parity/equivalence target, which deliberately compiles barretenberg headers. +include_directories( + ${CMAKE_CURRENT_SOURCE_DIR}/src + ${LMDBLIB_SRC} + ${MSGPACK_INCLUDE} + ${LMDB_INCLUDE} + ${IPC_RUNTIME_SRC} +) + +# --------------------------------------------------------------------------- +# Codegen the wsdb server dispatch from the local schema (this package owns the +# wire contract). bb's client codegen reads the same schema from this path. +# --------------------------------------------------------------------------- +set(WSDB_SCHEMA ${CMAKE_CURRENT_SOURCE_DIR}/../wsdb_schema.jsonc) +set(WSDB_GEN_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/wsdb/generated) +set(WSDB_GEN_OUTPUTS + ${WSDB_GEN_DIR}/wsdb_dispatch.hpp + ${WSDB_GEN_DIR}/wsdb_ipc_server.hpp + ${WSDB_GEN_DIR}/wsdb_types.hpp + ${WSDB_GEN_DIR}/ipc_codegen/msgpack_adaptor.hpp + ${WSDB_GEN_DIR}/ipc_codegen/named_union.hpp + ${WSDB_GEN_DIR}/ipc_codegen/throw.hpp +) +file(GLOB_RECURSE IPC_CODEGEN_SRC + ${IPC_CODEGEN_DIR}/src/*.ts + ${IPC_CODEGEN_DIR}/templates/cpp/*.hpp +) +add_custom_command( + OUTPUT ${WSDB_GEN_OUTPUTS} + COMMAND node --experimental-strip-types --experimental-transform-types --no-warnings + ${IPC_CODEGEN_DIR}/src/generate.ts + --schema ${WSDB_SCHEMA} + --lang cpp + --out ${WSDB_GEN_DIR} + --server + --cpp-namespace azteclabs::wsdb + --cpp-include-dir wsdb/generated + DEPENDS ${WSDB_SCHEMA} ${IPC_CODEGEN_SRC} + COMMENT "Generating WSDB IPC server dispatch from wsdb_schema.jsonc" + VERBATIM +) +add_custom_target(wsdb_ipc_server_generated DEPENDS ${WSDB_GEN_OUTPUTS}) + +# --------------------------------------------------------------------------- +# CLI11 (header-only) for aztec-wsdb command-line parsing. Fetched and pinned to +# v2.4.2 (matches barretenberg's vendored copy) rather than vendored in-tree. +# --------------------------------------------------------------------------- +include(FetchContent) +set(CLI11_BUILD_TESTS OFF CACHE INTERNAL "") +set(CLI11_BUILD_EXAMPLES OFF CACHE INTERNAL "") +set(CLI11_BUILD_DOCS OFF CACHE INTERNAL "") +FetchContent_Declare( + cli11 + GIT_REPOSITORY https://github.com/CLIUtils/CLI11.git + GIT_TAG v2.4.2 + GIT_SHALLOW TRUE +) +FetchContent_MakeAvailable(cli11) + +# --------------------------------------------------------------------------- +# wsdb core: world_state + persistent merkle storage + server handlers. +# --------------------------------------------------------------------------- +set(WSDB_CORE_SOURCES + src/common/thread_pool.cpp + src/world_state/types.cpp + src/world_state/world_state.cpp + src/merkle_tree/lmdb_store/lmdb_tree_store.cpp + src/merkle_tree/merkle_tree_id.cpp + src/wsdb/wsdb_handlers.cpp + src/wsdb/wsdb_ipc_server.cpp + src/wsdb/cli.cpp +) +add_library(wsdb_core STATIC ${WSDB_CORE_SOURCES}) +add_dependencies(wsdb_core wsdb_ipc_server_generated) +# Wrap libbarretenberg.a + libenv.a in --start-group/--end-group: bb's archive +# references env symbols (logstr, throw_or_abort_impl) and the link order between +# the two is not a clean DAG, so let the linker resolve them mutually. +target_link_libraries(wsdb_core PUBLIC + -Wl,--start-group ${BB_LIB} ${ENV_LIB} ${LMDBLIB_LIB} -Wl,--end-group + ${IPC_RUNTIME_LIB} ${LMDB_LIB} pthread) +# CLI11 is only used in cli.cpp's implementation, so keep it private to wsdb_core. +target_link_libraries(wsdb_core PRIVATE CLI11::CLI11) + +# --------------------------------------------------------------------------- +# aztec-wsdb binary. +# --------------------------------------------------------------------------- +add_executable(aztec-wsdb src/wsdb/main.cpp) +target_link_libraries(aztec-wsdb PRIVATE wsdb_core) + +# --------------------------------------------------------------------------- +# Tests. The equality test links MemoryMerkleDB (the reference impl, which stays +# in bb's vm2_sim module -> libvm2_sim.a) against a real WorldState (local). It +# can no longer live in vm2_tests since that target no longer links world_state. +# --------------------------------------------------------------------------- +option(WSDB_BUILD_TESTS "Build wsdb C++ tests" ON) +if(WSDB_BUILD_TESTS) + include(ExternalProject) + set(VM2_SIM_LIB "${BB_BUILD}/lib/libvm2_sim.a") + set(GTEST_LIB "${BB_BUILD}/lib/libgtest.a") + set(GMOCK_LIB "${BB_BUILD}/lib/libgmock.a") + set(GMOCK_MAIN_LIB "${BB_BUILD}/lib/libgmock_main.a") + # gtest/gmock libs come from bb's published archives (cached), but their headers + # live under bb's uncached _deps. Fetch our own copy, pinned to bb's tag (cmake/gtest.cmake). + set(GTEST_PREFIX "${DEPS_PREFIX}/gtest-src") + ExternalProject_Add( + gtest_headers + PREFIX ${GTEST_PREFIX} + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG v1.13.0 + GIT_SHALLOW TRUE + SOURCE_DIR ${GTEST_PREFIX}/src/gtest + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + UPDATE_COMMAND "" + ) + set(GTEST_INCLUDE "${GTEST_PREFIX}/src/gtest/googletest/include") + set(GMOCK_INCLUDE "${GTEST_PREFIX}/src/gtest/googlemock/include") + + # The decoupled test set: exercises wsdb's own tree/storage with no barretenberg + # headers. (field_element parity + memory_merkle_db equivalence tests link bb and + # live in their own target below.) + set(WSDB_TEST_SOURCES + src/common/uint256.test.cpp + src/merkle_tree/fixtures.test.cpp + src/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp + src/merkle_tree/node_store/content_addressed_cache.test.cpp + src/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp + src/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp + src/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp + src/world_state/world_state.test.cpp + ) + add_executable(wsdb_tests ${WSDB_TEST_SOURCES}) + add_dependencies(wsdb_tests wsdb_ipc_server_generated gtest_headers) + target_include_directories(wsdb_tests PRIVATE ${GTEST_INCLUDE} ${GMOCK_INCLUDE}) + # wsdb_core (WorldState + persistent merkle), vm2_sim (MemoryMerkleDB) and bb's + # archive reference each other's symbols, so group them all for mutual resolution. + target_link_libraries(wsdb_tests PRIVATE + -Wl,--start-group + $ ${VM2_SIM_LIB} ${BB_LIB} ${ENV_LIB} ${LMDBLIB_LIB} + -Wl,--end-group + ${IPC_RUNTIME_LIB} ${LMDB_LIB} pthread + ${GMOCK_MAIN_LIB} ${GMOCK_LIB} ${GTEST_LIB} + ) + add_dependencies(wsdb_tests wsdb_core) +endif() + +# --------------------------------------------------------------------------- +# Barretenberg-linked tests (OFF by default; require a prebuilt barretenberg). +# These deliberately compile bb headers, so they add BB_SRC back and define +# +# - wsdb_parity_tests: proves FieldElement + the poseidon2 c_bind are byte-identical +# to bb::fr / bb's Poseidon2HashPolicy (the consensus gate). +# - The MemoryMerkleDB equivalence test (src/wsdb/memory_merkle_db.test.cpp) is NOT +# wired here yet: it includes both wsdb's forked crypto::merkle_tree headers and +# barretenberg's real ones in one TU, which collide. Reviving it needs wsdb's merkle +# types moved to a dedicated namespace. Left in-tree (may rot) per design. +# --------------------------------------------------------------------------- +option(WSDB_BUILD_BB_TESTS "Build the barretenberg-linked parity/equivalence tests" OFF) +if(WSDB_BUILD_BB_TESTS AND WSDB_BUILD_TESTS) + # These tests compile barretenberg headers, which include tracy; fetch its headers. + set(TRACY_SOURCE_DIR "${DEPS_PREFIX}/tracy-src") + execute_process( + COMMAND sh -c "mkdir -p ${TRACY_SOURCE_DIR} && cd ${TRACY_SOURCE_DIR} && git init --quiet && (git remote add origin https://github.com/wolfpld/tracy.git 2>/dev/null || true) && git fetch --depth 1 origin --quiet 5d542dc09f3d9378d005092a4ad446bd405f819a && git reset --quiet --hard FETCH_HEAD" + RESULT_VARIABLE tracy_result + ) + if(tracy_result) + message(FATAL_ERROR "Failed to download Tracy") + endif() + set(TRACY_INCLUDE "${TRACY_SOURCE_DIR}/public") + + add_executable(wsdb_parity_tests src/field/field_element.test.cpp) + add_dependencies(wsdb_parity_tests wsdb_ipc_server_generated gtest_headers) + target_include_directories(wsdb_parity_tests PRIVATE ${BB_SRC} ${TRACY_INCLUDE} ${GTEST_INCLUDE} ${GMOCK_INCLUDE}) + target_link_libraries(wsdb_parity_tests PRIVATE + -Wl,--start-group ${BB_LIB} ${ENV_LIB} ${LMDBLIB_LIB} -Wl,--end-group + ${IPC_RUNTIME_LIB} ${LMDB_LIB} pthread + ${GMOCK_MAIN_LIB} ${GMOCK_LIB} ${GTEST_LIB} + ) +endif() diff --git a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp b/native-packages/wsdb/cpp/src/benchmark/append_only_tree_bench/append_only_tree.bench.cpp similarity index 71% rename from barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp rename to native-packages/wsdb/cpp/src/benchmark/append_only_tree_bench/append_only_tree.bench.cpp index ed54fd5debc5..63f2643b1594 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp +++ b/native-packages/wsdb/cpp/src/benchmark/append_only_tree_bench/append_only_tree.bench.cpp @@ -1,25 +1,24 @@ -#include "barretenberg/common/thread_pool.hpp" -#include "barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" -#include "barretenberg/crypto/merkle_tree/fixtures.hpp" -#include "barretenberg/crypto/merkle_tree/hash.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/array_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/response.hpp" -#include "barretenberg/crypto/merkle_tree/signal.hpp" -#include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/numeric/random/engine.hpp" +#include "common/thread_pool.hpp" +#include "merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" +#include "merkle_tree/fixtures.hpp" +#include "field/hash_policy.hpp" +#include "merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/node_store/array_store.hpp" +#include "merkle_tree/node_store/cached_content_addressed_tree_store.hpp" +#include "merkle_tree/response.hpp" +#include "merkle_tree/signal.hpp" +#include "field/field_element.hpp" #include #include #include #include using namespace benchmark; -using namespace bb::crypto::merkle_tree; +using namespace azteclabs::wsdb::merkle_tree; namespace { -using StoreType = ContentAddressedCachedTreeStore; +using StoreType = ContentAddressedCachedTreeStore; using Poseidon2 = ContentAddressedAppendOnlyTree; diff --git a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp b/native-packages/wsdb/cpp/src/benchmark/indexed_tree_bench/indexed_tree.bench.cpp similarity index 95% rename from barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp rename to native-packages/wsdb/cpp/src/benchmark/indexed_tree_bench/indexed_tree.bench.cpp index ec649651882e..f8c748f3b6f7 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp +++ b/native-packages/wsdb/cpp/src/benchmark/indexed_tree_bench/indexed_tree.bench.cpp @@ -1,19 +1,18 @@ -#include "barretenberg/common/thread_pool.hpp" -#include "barretenberg/crypto/merkle_tree/fixtures.hpp" -#include "barretenberg/crypto/merkle_tree/hash.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/response.hpp" -#include "barretenberg/numeric/random/engine.hpp" +#include "common/thread_pool.hpp" +#include "merkle_tree/fixtures.hpp" +#include "field/hash_policy.hpp" +#include "merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp" +#include "merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/node_store/cached_content_addressed_tree_store.hpp" +#include "merkle_tree/response.hpp" #include #include #include #include using namespace benchmark; -using namespace bb::crypto::merkle_tree; +using namespace azteclabs::wsdb::merkle_tree; using StoreType = ContentAddressedCachedTreeStore; diff --git a/native-packages/wsdb/cpp/src/common/assert.hpp b/native-packages/wsdb/cpp/src/common/assert.hpp new file mode 100644 index 000000000000..e3fe39fd5eae --- /dev/null +++ b/native-packages/wsdb/cpp/src/common/assert.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include "log.hpp" + +// Minimal assertion helpers for wsdb, replacing barretenberg's common/assert.hpp with no bb +// dependency. Assertions throw (the wsdb service turns these into error responses rather than +// aborting the process). Unlike barretenberg there is no throw_or_abort indirection — that exists +// to support wasm (which cannot throw); wsdb is a native server, so it just throws. +#define BB_ASSERT_BINOP(a, op, b) \ + do { \ + if (!((a)op(b))) { \ + throw std::runtime_error( \ + azteclabs::wsdb::detail::log_concat("assertion failed: ", #a, " ", #op, " ", #b)); \ + } \ + } while (0) + +#define BB_ASSERT_GT(a, b, ...) BB_ASSERT_BINOP(a, >, b) +#define BB_ASSERT_GTE(a, b, ...) BB_ASSERT_BINOP(a, >=, b) +#define BB_ASSERT_LT(a, b, ...) BB_ASSERT_BINOP(a, <, b) +#define BB_ASSERT_LTE(a, b, ...) BB_ASSERT_BINOP(a, <=, b) +#define BB_ASSERT_EQ(a, b, ...) BB_ASSERT_BINOP(a, ==, b) +#define BB_ASSERT(cond, ...) \ + do { \ + if (!(cond)) { \ + throw std::runtime_error(azteclabs::wsdb::detail::log_concat("assertion failed: ", #cond)); \ + } \ + } while (0) +#define BB_ASSERT_DEBUG(...) ((void)0) diff --git a/native-packages/wsdb/cpp/src/common/aztec_constants.hpp b/native-packages/wsdb/cpp/src/common/aztec_constants.hpp new file mode 100644 index 000000000000..2d24f41f87dd --- /dev/null +++ b/native-packages/wsdb/cpp/src/common/aztec_constants.hpp @@ -0,0 +1,277 @@ +// GENERATED FILE - DO NOT EDIT, RUN yarn remake-constants in yarn-project/constants +#pragma once + +#define MAX_ETH_ADDRESS_BIT_SIZE 160 +#define MAX_ETH_ADDRESS_VALUE "0x000000000000000000000000ffffffffffffffffffffffffffffffffffffffff" +#define ARCHIVE_HEIGHT 30 +#define NOTE_HASH_TREE_HEIGHT 42 +#define PUBLIC_DATA_TREE_HEIGHT 40 +#define NULLIFIER_TREE_HEIGHT 42 +#define L1_TO_L2_MSG_TREE_HEIGHT 36 +#define NOTE_HASH_TREE_LEAF_COUNT 0x40000000000 +#define L1_TO_L2_MSG_TREE_LEAF_COUNT 0x1000000000 +#define NULLIFIER_SUBTREE_HEIGHT 6 +#define MAX_NOTE_HASHES_PER_TX 64 +#define MAX_NULLIFIERS_PER_TX 64 +#define MAX_ENQUEUED_CALLS_PER_TX 32 +#define MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX 64 +#define MAX_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX 63 +#define MAX_L2_TO_L1_MSGS_PER_TX 8 +#define GENESIS_BLOCK_HEADER_HASH "0x0e7bf88e8833c27b0fca6be614ecc2c63def7bffa35b3ccfa1caff3e39e0c0d2" +#define GENESIS_ARCHIVE_ROOT "0x177a4955b31ecaafad999753938a44e526b54c5ba5d536688227f85f15cfbdf5" +#define MAX_PACKED_PUBLIC_BYTECODE_SIZE_IN_FIELDS 3000 +#define MAX_PROTOCOL_CONTRACTS 11 +#define CONTRACT_CLASS_REGISTRY_CONTRACT_ADDRESS 1 +#define CONTRACT_INSTANCE_REGISTRY_CONTRACT_ADDRESS 2 +#define FEE_JUICE_ADDRESS 3 +#define FEE_JUICE_BALANCES_SLOT 1 +#define UPDATED_CLASS_IDS_SLOT 1 +#define FLAT_PUBLIC_LOGS_HEADER_LENGTH 1 +#define FLAT_PUBLIC_LOGS_PAYLOAD_LENGTH 4096 +#define PUBLIC_LOGS_LENGTH 4097 +#define PUBLIC_LOG_HEADER_LENGTH 2 +#define MAX_PUBLIC_LOG_SIZE_IN_FIELDS 4094 +#define AVM_ACCUMULATED_DATA_LENGTH 4377 +#define AVM_CIRCUIT_PUBLIC_INPUTS_LENGTH 5008 +#define AVM_VERIFICATION_KEY_LENGTH_IN_FIELDS 86 +#define MAX_PUBLIC_CALLS_TO_UNIQUE_CONTRACT_CLASS_IDS 21 +#define MEM_TAG_FF 0 +#define MEM_TAG_U1 1 +#define MEM_TAG_U8 2 +#define MEM_TAG_U16 3 +#define MEM_TAG_U32 4 +#define MEM_TAG_U64 5 +#define MEM_TAG_U128 6 +#define AVM_HIGHEST_MEM_ADDRESS 0xffffffff +#define AVM_MEMORY_NUM_BITS 32 +#define AVM_MEMORY_SIZE 0x100000000 +#define AVM_BITWISE_AND_OP_ID 1 +#define AVM_BITWISE_OR_OP_ID 2 +#define AVM_BITWISE_XOR_OP_ID 4 +#define AVM_KECCAKF1600_NUM_ROUNDS 24 +#define AVM_KECCAKF1600_STATE_SIZE 25 +#define AVM_TX_PHASE_VALUE_START 0 +#define AVM_TX_PHASE_VALUE_SETUP 3 +#define AVM_TX_PHASE_VALUE_LAST 11 +#define AVM_SUBTRACE_ID_EXECUTION 1 +#define AVM_SUBTRACE_ID_ALU 2 +#define AVM_SUBTRACE_ID_BITWISE 4 +#define AVM_SUBTRACE_ID_CAST 8 +#define AVM_SUBTRACE_ID_CALLDATA_COPY 16 +#define AVM_SUBTRACE_ID_RETURNDATA_COPY 32 +#define AVM_SUBTRACE_ID_SET 64 +#define AVM_SUBTRACE_ID_GETCONTRACTINSTANCE 128 +#define AVM_SUBTRACE_ID_EMITPUBLICLOG 256 +#define AVM_SUBTRACE_ID_POSEIDON2_PERM 512 +#define AVM_SUBTRACE_ID_SHA256_COMPRESSION 1024 +#define AVM_SUBTRACE_ID_KECCAKF1600 2048 +#define AVM_SUBTRACE_ID_ECC 4096 +#define AVM_SUBTRACE_ID_TO_RADIX 8192 +#define AVM_DYN_GAS_ID_CALLDATACOPY 1 +#define AVM_DYN_GAS_ID_RETURNDATACOPY 2 +#define AVM_DYN_GAS_ID_TORADIX 4 +#define AVM_DYN_GAS_ID_BITWISE 8 +#define AVM_DYN_GAS_ID_EMITPUBLICLOG 16 +#define AVM_DYN_GAS_ID_SSTORE 32 +#define AVM_EXEC_OP_ID_GETENVVAR 1 +#define AVM_EXEC_OP_ID_MOV 2 +#define AVM_EXEC_OP_ID_JUMP 4 +#define AVM_EXEC_OP_ID_JUMPI 8 +#define AVM_EXEC_OP_ID_CALL 16 +#define AVM_EXEC_OP_ID_STATICCALL 32 +#define AVM_EXEC_OP_ID_INTERNALCALL 64 +#define AVM_EXEC_OP_ID_INTERNALRETURN 128 +#define AVM_EXEC_OP_ID_RETURN 256 +#define AVM_EXEC_OP_ID_REVERT 512 +#define AVM_EXEC_OP_ID_SUCCESSCOPY 1024 +#define AVM_EXEC_OP_ID_RETURNDATASIZE 2048 +#define AVM_EXEC_OP_ID_DEBUGLOG 4096 +#define AVM_EXEC_OP_ID_SLOAD 8192 +#define AVM_EXEC_OP_ID_SSTORE 16384 +#define AVM_EXEC_OP_ID_NOTEHASH_EXISTS 32768 +#define AVM_EXEC_OP_ID_EMIT_NOTEHASH 65536 +#define AVM_EXEC_OP_ID_L1_TO_L2_MESSAGE_EXISTS 131072 +#define AVM_EXEC_OP_ID_NULLIFIER_EXISTS 262144 +#define AVM_EXEC_OP_ID_EMIT_NULLIFIER 524288 +#define AVM_EXEC_OP_ID_SENDL2TOL1MSG 1048576 +#define AVM_EXEC_OP_ID_ALU_ADD 1 +#define AVM_EXEC_OP_ID_ALU_SUB 2 +#define AVM_EXEC_OP_ID_ALU_MUL 4 +#define AVM_EXEC_OP_ID_ALU_DIV 8 +#define AVM_EXEC_OP_ID_ALU_FDIV 16 +#define AVM_EXEC_OP_ID_ALU_EQ 32 +#define AVM_EXEC_OP_ID_ALU_LT 64 +#define AVM_EXEC_OP_ID_ALU_LTE 128 +#define AVM_EXEC_OP_ID_ALU_NOT 256 +#define AVM_EXEC_OP_ID_ALU_SHL 512 +#define AVM_EXEC_OP_ID_ALU_SHR 1024 +#define AVM_EXEC_OP_ID_ALU_TRUNCATE 2048 +#define AVM_PUBLIC_INPUTS_GLOBAL_VARIABLES_ROW_IDX 0 +#define AVM_PUBLIC_INPUTS_GLOBAL_VARIABLES_CHAIN_ID_ROW_IDX 0 +#define AVM_PUBLIC_INPUTS_GLOBAL_VARIABLES_VERSION_ROW_IDX 1 +#define AVM_PUBLIC_INPUTS_GLOBAL_VARIABLES_BLOCK_NUMBER_ROW_IDX 2 +#define AVM_PUBLIC_INPUTS_GLOBAL_VARIABLES_SLOT_NUMBER_ROW_IDX 3 +#define AVM_PUBLIC_INPUTS_GLOBAL_VARIABLES_TIMESTAMP_ROW_IDX 4 +#define AVM_PUBLIC_INPUTS_GLOBAL_VARIABLES_COINBASE_ROW_IDX 5 +#define AVM_PUBLIC_INPUTS_GLOBAL_VARIABLES_FEE_RECIPIENT_ROW_IDX 6 +#define AVM_PUBLIC_INPUTS_GLOBAL_VARIABLES_GAS_FEES_ROW_IDX 7 +#define AVM_PUBLIC_INPUTS_PROTOCOL_CONTRACTS_ROW_IDX 8 +#define AVM_PUBLIC_INPUTS_START_TREE_SNAPSHOTS_ROW_IDX 19 +#define AVM_PUBLIC_INPUTS_START_TREE_SNAPSHOTS_L1_TO_L2_MESSAGE_TREE_ROW_IDX 19 +#define AVM_PUBLIC_INPUTS_START_TREE_SNAPSHOTS_NOTE_HASH_TREE_ROW_IDX 20 +#define AVM_PUBLIC_INPUTS_START_TREE_SNAPSHOTS_NULLIFIER_TREE_ROW_IDX 21 +#define AVM_PUBLIC_INPUTS_START_TREE_SNAPSHOTS_PUBLIC_DATA_TREE_ROW_IDX 22 +#define AVM_PUBLIC_INPUTS_START_GAS_USED_ROW_IDX 23 +#define AVM_PUBLIC_INPUTS_GAS_SETTINGS_ROW_IDX 24 +#define AVM_PUBLIC_INPUTS_GAS_SETTINGS_GAS_LIMITS_ROW_IDX 24 +#define AVM_PUBLIC_INPUTS_GAS_SETTINGS_TEARDOWN_GAS_LIMITS_ROW_IDX 25 +#define AVM_PUBLIC_INPUTS_GAS_SETTINGS_MAX_FEES_PER_GAS_ROW_IDX 26 +#define AVM_PUBLIC_INPUTS_GAS_SETTINGS_MAX_PRIORITY_FEES_PER_GAS_ROW_IDX 27 +#define AVM_PUBLIC_INPUTS_EFFECTIVE_GAS_FEES_ROW_IDX 28 +#define AVM_PUBLIC_INPUTS_FEE_PAYER_ROW_IDX 29 +#define AVM_PUBLIC_INPUTS_PROVER_ID_ROW_IDX 30 +#define AVM_PUBLIC_INPUTS_PUBLIC_CALL_REQUEST_ARRAY_LENGTHS_ROW_IDX 31 +#define AVM_PUBLIC_INPUTS_PUBLIC_CALL_REQUEST_ARRAY_LENGTHS_SETUP_CALLS_ROW_IDX 31 +#define AVM_PUBLIC_INPUTS_PUBLIC_CALL_REQUEST_ARRAY_LENGTHS_APP_LOGIC_CALLS_ROW_IDX 32 +#define AVM_PUBLIC_INPUTS_PUBLIC_CALL_REQUEST_ARRAY_LENGTHS_TEARDOWN_CALL_ROW_IDX 33 +#define AVM_PUBLIC_INPUTS_PUBLIC_SETUP_CALL_REQUESTS_ROW_IDX 34 +#define AVM_PUBLIC_INPUTS_PUBLIC_APP_LOGIC_CALL_REQUESTS_ROW_IDX 66 +#define AVM_PUBLIC_INPUTS_PUBLIC_TEARDOWN_CALL_REQUEST_ROW_IDX 98 +#define AVM_PUBLIC_INPUTS_PREVIOUS_NON_REVERTIBLE_ACCUMULATED_DATA_ARRAY_LENGTHS_ROW_IDX 99 +#define AVM_PUBLIC_INPUTS_PREVIOUS_NON_REVERTIBLE_ACCUMULATED_DATA_ARRAY_LENGTHS_NOTE_HASHES_ROW_IDX 99 +#define AVM_PUBLIC_INPUTS_PREVIOUS_NON_REVERTIBLE_ACCUMULATED_DATA_ARRAY_LENGTHS_NULLIFIERS_ROW_IDX 100 +#define AVM_PUBLIC_INPUTS_PREVIOUS_NON_REVERTIBLE_ACCUMULATED_DATA_ARRAY_LENGTHS_L2_TO_L1_MSGS_ROW_IDX 101 +#define AVM_PUBLIC_INPUTS_PREVIOUS_REVERTIBLE_ACCUMULATED_DATA_ARRAY_LENGTHS_ROW_IDX 102 +#define AVM_PUBLIC_INPUTS_PREVIOUS_REVERTIBLE_ACCUMULATED_DATA_ARRAY_LENGTHS_NOTE_HASHES_ROW_IDX 102 +#define AVM_PUBLIC_INPUTS_PREVIOUS_REVERTIBLE_ACCUMULATED_DATA_ARRAY_LENGTHS_NULLIFIERS_ROW_IDX 103 +#define AVM_PUBLIC_INPUTS_PREVIOUS_REVERTIBLE_ACCUMULATED_DATA_ARRAY_LENGTHS_L2_TO_L1_MSGS_ROW_IDX 104 +#define AVM_PUBLIC_INPUTS_PREVIOUS_NON_REVERTIBLE_ACCUMULATED_DATA_ROW_IDX 105 +#define AVM_PUBLIC_INPUTS_PREVIOUS_NON_REVERTIBLE_ACCUMULATED_DATA_NOTE_HASHES_ROW_IDX 105 +#define AVM_PUBLIC_INPUTS_PREVIOUS_NON_REVERTIBLE_ACCUMULATED_DATA_NULLIFIERS_ROW_IDX 169 +#define AVM_PUBLIC_INPUTS_PREVIOUS_NON_REVERTIBLE_ACCUMULATED_DATA_L2_TO_L1_MSGS_ROW_IDX 233 +#define AVM_PUBLIC_INPUTS_PREVIOUS_REVERTIBLE_ACCUMULATED_DATA_ROW_IDX 241 +#define AVM_PUBLIC_INPUTS_PREVIOUS_REVERTIBLE_ACCUMULATED_DATA_NOTE_HASHES_ROW_IDX 241 +#define AVM_PUBLIC_INPUTS_PREVIOUS_REVERTIBLE_ACCUMULATED_DATA_NULLIFIERS_ROW_IDX 305 +#define AVM_PUBLIC_INPUTS_PREVIOUS_REVERTIBLE_ACCUMULATED_DATA_L2_TO_L1_MSGS_ROW_IDX 369 +#define AVM_PUBLIC_INPUTS_END_TREE_SNAPSHOTS_ROW_IDX 377 +#define AVM_PUBLIC_INPUTS_END_TREE_SNAPSHOTS_L1_TO_L2_MESSAGE_TREE_ROW_IDX 377 +#define AVM_PUBLIC_INPUTS_END_TREE_SNAPSHOTS_NOTE_HASH_TREE_ROW_IDX 378 +#define AVM_PUBLIC_INPUTS_END_TREE_SNAPSHOTS_NULLIFIER_TREE_ROW_IDX 379 +#define AVM_PUBLIC_INPUTS_END_TREE_SNAPSHOTS_PUBLIC_DATA_TREE_ROW_IDX 380 +#define AVM_PUBLIC_INPUTS_END_GAS_USED_ROW_IDX 381 +#define AVM_PUBLIC_INPUTS_AVM_ACCUMULATED_DATA_ARRAY_LENGTHS_ROW_IDX 382 +#define AVM_PUBLIC_INPUTS_AVM_ACCUMULATED_DATA_ARRAY_LENGTHS_NOTE_HASHES_ROW_IDX 382 +#define AVM_PUBLIC_INPUTS_AVM_ACCUMULATED_DATA_ARRAY_LENGTHS_NULLIFIERS_ROW_IDX 383 +#define AVM_PUBLIC_INPUTS_AVM_ACCUMULATED_DATA_ARRAY_LENGTHS_L2_TO_L1_MSGS_ROW_IDX 384 +#define AVM_PUBLIC_INPUTS_AVM_ACCUMULATED_DATA_ARRAY_LENGTHS_PUBLIC_DATA_WRITES_ROW_IDX 385 +#define AVM_PUBLIC_INPUTS_AVM_ACCUMULATED_DATA_ROW_IDX 386 +#define AVM_PUBLIC_INPUTS_AVM_ACCUMULATED_DATA_NOTE_HASHES_ROW_IDX 386 +#define AVM_PUBLIC_INPUTS_AVM_ACCUMULATED_DATA_NULLIFIERS_ROW_IDX 450 +#define AVM_PUBLIC_INPUTS_AVM_ACCUMULATED_DATA_L2_TO_L1_MSGS_ROW_IDX 514 +#define AVM_PUBLIC_INPUTS_AVM_ACCUMULATED_DATA_PUBLIC_LOGS_ROW_IDX 522 +#define AVM_PUBLIC_INPUTS_AVM_ACCUMULATED_DATA_PUBLIC_DATA_WRITES_ROW_IDX 4619 +#define AVM_PUBLIC_INPUTS_TRANSACTION_FEE_ROW_IDX 4683 +#define AVM_PUBLIC_INPUTS_REVERTED_ROW_IDX 4684 +#define AVM_PUBLIC_INPUTS_COLUMNS_MAX_LENGTH 4685 +#define AVM_NUM_PUBLIC_INPUT_COLUMNS 4 +#define AVM_PUBLIC_INPUTS_COLUMNS_COMBINED_LENGTH 18740 +#define AVM_V2_PROOF_LENGTH_IN_FIELDS_PADDED 16400 +#define AVM_V2_VERIFICATION_KEY_LENGTH_IN_FIELDS_PADDED 1000 +#define TX_DA_GAS_OVERHEAD 96 +#define PUBLIC_TX_L2_GAS_OVERHEAD 540000 +#define AVM_MAX_PROCESSABLE_L2_GAS 6000000 +#define MAX_PROCESSABLE_L2_GAS 6540000 +#define AVM_PC_SIZE_IN_BITS 32 +#define AVM_MAX_OPERANDS 7 +#define AVM_MAX_REGISTERS 6 +#define AVM_ADDRESSING_BASE_RESOLUTION_L2_GAS 3 +#define AVM_ADDRESSING_INDIRECT_L2_GAS 3 +#define AVM_ADDRESSING_RELATIVE_L2_GAS 3 +#define AVM_ADD_BASE_L2_GAS 12 +#define AVM_SUB_BASE_L2_GAS 12 +#define AVM_MUL_BASE_L2_GAS 27 +#define AVM_DIV_BASE_L2_GAS 27 +#define AVM_FDIV_BASE_L2_GAS 225 +#define AVM_EQ_BASE_L2_GAS 12 +#define AVM_LT_BASE_L2_GAS 42 +#define AVM_LTE_BASE_L2_GAS 42 +#define AVM_AND_BASE_L2_GAS 12 +#define AVM_OR_BASE_L2_GAS 12 +#define AVM_XOR_BASE_L2_GAS 12 +#define AVM_NOT_BASE_L2_GAS 12 +#define AVM_SHL_BASE_L2_GAS 18 +#define AVM_SHR_BASE_L2_GAS 18 +#define AVM_CAST_BASE_L2_GAS 27 +#define AVM_GETENVVAR_BASE_L2_GAS 12 +#define AVM_CALLDATACOPY_BASE_L2_GAS 18 +#define AVM_SUCCESSCOPY_BASE_L2_GAS 12 +#define AVM_RETURNDATASIZE_BASE_L2_GAS 12 +#define AVM_RETURNDATACOPY_BASE_L2_GAS 18 +#define AVM_JUMP_BASE_L2_GAS 9 +#define AVM_JUMPI_BASE_L2_GAS 9 +#define AVM_INTERNALCALL_BASE_L2_GAS 9 +#define AVM_INTERNALRETURN_BASE_L2_GAS 9 +#define AVM_SET_BASE_L2_GAS 27 +#define AVM_MOV_BASE_L2_GAS 12 +#define AVM_SLOAD_BASE_L2_GAS 1290 +#define AVM_SSTORE_BASE_L2_GAS 33140 +#define AVM_NOTEHASHEXISTS_BASE_L2_GAS 504 +#define AVM_EMITNOTEHASH_BASE_L2_GAS 19275 +#define AVM_NULLIFIEREXISTS_BASE_L2_GAS 903 +#define AVM_EMITNULLIFIER_BASE_L2_GAS 30800 +#define AVM_L1TOL2MSGEXISTS_BASE_L2_GAS 540 +#define AVM_GETCONTRACTINSTANCE_BASE_L2_GAS 6108 +#define AVM_EMITPUBLICLOG_BASE_L2_GAS 15 +#define AVM_SENDL2TOL1MSG_BASE_L2_GAS 5239 +#define AVM_CALL_BASE_L2_GAS 9936 +#define AVM_STATICCALL_BASE_L2_GAS 9936 +#define AVM_RETURN_BASE_L2_GAS 9 +#define AVM_REVERT_BASE_L2_GAS 9 +#define AVM_DEBUGLOG_BASE_L2_GAS 9 +#define AVM_POSEIDON2_BASE_L2_GAS 360 +#define AVM_SHA256COMPRESSION_BASE_L2_GAS 12288 +#define AVM_KECCAKF1600_BASE_L2_GAS 58176 +#define AVM_ECADD_BASE_L2_GAS 180 +#define AVM_TORADIXBE_BASE_L2_GAS 24 +#define AVM_CALLDATACOPY_DYN_L2_GAS 3 +#define AVM_RETURNDATACOPY_DYN_L2_GAS 3 +#define AVM_TORADIXBE_DYN_L2_GAS 3 +#define AVM_BITWISE_DYN_L2_GAS 3 +#define AVM_EMITPUBLICLOG_DYN_L2_GAS 3 +#define AVM_EMITNOTEHASH_BASE_DA_GAS 32 +#define AVM_EMITNULLIFIER_BASE_DA_GAS 32 +#define AVM_SENDL2TOL1MSG_BASE_DA_GAS 32 +#define AVM_EMITPUBLICLOG_BASE_DA_GAS 64 +#define AVM_EMITPUBLICLOG_DYN_DA_GAS 32 +#define AVM_SSTORE_DYN_DA_GAS 64 +#define AVM_WRITTEN_PUBLIC_DATA_SLOTS_TREE_HEIGHT 6 +#define AVM_WRITTEN_PUBLIC_DATA_SLOTS_TREE_INITIAL_ROOT \ + "0x13954b76f8ab76c38ab687731f802fc90e5d55e0592fa6f56d0a73024b3f8356" +#define AVM_WRITTEN_PUBLIC_DATA_SLOTS_TREE_INITIAL_SIZE 1 +#define AVM_RETRIEVED_BYTECODES_TREE_HEIGHT 5 +#define AVM_RETRIEVED_BYTECODES_TREE_INITIAL_ROOT "0x0ff65ad321f05745d4c80e2e18e94241093d2e49017ff8cc151df4f207a43c12" +#define AVM_RETRIEVED_BYTECODES_TREE_INITIAL_SIZE 1 +#define TIMESTAMP_OF_CHANGE_BIT_SIZE 32 +#define UPDATES_DELAYED_PUBLIC_MUTABLE_VALUES_LEN 3 +#define UPDATES_DELAYED_PUBLIC_MUTABLE_METADATA_BIT_SIZE 144 +#define DEFAULT_MAX_DEBUG_LOG_MEMORY_READS 125000 +#define DOM_SEP__SILOED_NOTE_HASH 3361878420UL +#define DOM_SEP__UNIQUE_NOTE_HASH 226850429UL +#define DOM_SEP__NOTE_HASH_NONCE 1721808740UL +#define DOM_SEP__SILOED_NULLIFIER 57496191UL +#define DOM_SEP__PUBLIC_LEAF_SLOT 1247650290UL +#define DOM_SEP__PUBLIC_STORAGE_MAP_SLOT 4015149901UL +#define DOM_SEP__MERKLE_HASH 2982624097UL +#define DOM_SEP__NULLIFIER_MERKLE 1157584160UL +#define DOM_SEP__PUBLIC_DATA_MERKLE 3756303423UL +#define DOM_SEP__WRITTEN_SLOTS_MERKLE 2292766212UL +#define DOM_SEP__RETRIEVED_BYTECODES_MERKLE 2789215184UL +#define DOM_SEP__PUBLIC_BYTECODE 260313585UL +#define DOM_SEP__CONTRACT_CLASS_ID 3923495515UL +#define DOM_SEP__SALTED_INITIALIZATION_HASH 2763052992UL +#define DOM_SEP__PUBLIC_KEYS_HASH 777457226UL +#define DOM_SEP__SINGLE_PUBLIC_KEY_HASH 3452068255UL +#define DOM_SEP__PARTIAL_ADDRESS 2103633018UL +#define DOM_SEP__CONTRACT_ADDRESS_V2 4099338721UL +#define DOM_SEP__BLOCK_HEADER_HASH 4195546849UL +#define DOM_SEP__PUBLIC_CALLDATA 2760353947UL diff --git a/native-packages/wsdb/cpp/src/common/bitop.hpp b/native-packages/wsdb/cpp/src/common/bitop.hpp new file mode 100644 index 000000000000..ff10224c2f8d --- /dev/null +++ b/native-packages/wsdb/cpp/src/common/bitop.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include +#include + +// Minimal bit helpers replacing barretenberg/numeric/bitop/* (no bb dependency). +namespace azteclabs::wsdb::numeric { + +// Index of the most-significant set bit (matches barretenberg's get_msb for n > 0). +inline uint64_t get_msb(uint64_t n) +{ + return n == 0 ? 0 : 63 - static_cast(std::countl_zero(n)); +} + +// base^exp for small integer powers (matches barretenberg's pow64). +inline uint64_t pow64(uint64_t base, uint64_t exp) +{ + uint64_t result = 1; + for (uint64_t i = 0; i < exp; ++i) { + result *= base; + } + return result; +} + +} // namespace azteclabs::wsdb::numeric diff --git a/native-packages/wsdb/cpp/src/common/log.hpp b/native-packages/wsdb/cpp/src/common/log.hpp new file mode 100644 index 000000000000..0deec81e1a22 --- /dev/null +++ b/native-packages/wsdb/cpp/src/common/log.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include + +// Minimal logging for wsdb, replacing barretenberg's common/log.hpp with no bb dependency. +// Note: we deliberately do NOT define a free `format(...)` here — lmdblib already provides +// one (azteclabs::lmdblib::format), and the merkle/storage code pulls it in via lmdblib; defining +// a second `format` in namespace bb would make every call ambiguous. +namespace azteclabs::wsdb { + +namespace detail { +template std::string log_concat(Args&&... args) +{ + std::ostringstream os; + (os << ... << args); + return os.str(); +} +} // namespace detail + +template void info(Args&&... args) +{ + std::cout << detail::log_concat(std::forward(args)...) << std::endl; +} + +template void error(Args&&... args) +{ + std::cerr << detail::log_concat(std::forward(args)...) << std::endl; +} + +} // namespace azteclabs::wsdb diff --git a/native-packages/wsdb/cpp/src/common/random.hpp b/native-packages/wsdb/cpp/src/common/random.hpp new file mode 100644 index 000000000000..250fbc663734 --- /dev/null +++ b/native-packages/wsdb/cpp/src/common/random.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include "field/field_element.hpp" +#include +#include + +// Minimal RNG shim replacing barretenberg/numeric/random/engine.hpp for tests/fixtures. +namespace azteclabs::wsdb::numeric { +struct RandomEngine { + static std::mt19937_64& gen() + { + static thread_local std::mt19937_64 g{ std::random_device{}() }; + return g; + } + wsdb::FieldElement get_random_uint256() { return wsdb::FieldElement::random_element(); } + uint64_t get_random_uint64() { return gen()(); } + uint32_t get_random_uint32() { return static_cast(gen()()); } + uint8_t get_random_uint8() { return static_cast(gen()()); } +}; +inline RandomEngine& get_randomness() +{ + static RandomEngine e; + return e; +} +inline RandomEngine& get_debug_randomness() +{ + static RandomEngine e; + return e; +} +} // namespace azteclabs::wsdb::numeric diff --git a/native-packages/wsdb/cpp/src/common/thread_pool.cpp b/native-packages/wsdb/cpp/src/common/thread_pool.cpp new file mode 100644 index 000000000000..3577c20cbf9c --- /dev/null +++ b/native-packages/wsdb/cpp/src/common/thread_pool.cpp @@ -0,0 +1,76 @@ + +#ifndef NO_MULTITHREADING + +#include "common/thread_pool.hpp" +namespace azteclabs::wsdb { + +ThreadPool::ThreadPool(size_t num_threads) +{ + workers.reserve(num_threads); + for (size_t i = 0; i < num_threads; ++i) { + workers.emplace_back(&ThreadPool::worker_loop, this, i); + } +} + +ThreadPool::~ThreadPool() +{ + { + std::unique_lock lock(tasks_mutex); + stop = true; + } + condition.notify_all(); + for (auto& worker : workers) { + worker.join(); + } +} + +void ThreadPool::enqueue(const std::function& task) +{ + { + std::unique_lock lock(tasks_mutex); + tasks.push(task); + } + condition.notify_one(); +} + +void ThreadPool::wait() +{ + std::unique_lock lock(tasks_mutex); + finished_condition.wait(lock, [this] { return tasks.empty() && tasks_running == 0; }); +} + +void ThreadPool::worker_loop(size_t /*unused*/) +{ + // info("created worker ", worker_num); + while (true) { + std::function task; + { + std::unique_lock lock(tasks_mutex); + condition.wait(lock, [this] { return !tasks.empty() || stop; }); + + if (tasks.empty() && stop) { + break; + } + + task = tasks.front(); + tasks.pop(); + tasks_running++; + } + // info("worker ", worker_num, " processing a task!"); + task(); + // info("task done"); + { + std::unique_lock lock(tasks_mutex); + tasks_running--; + if (tasks.empty() && tasks_running == 0) { + // info("notifying main thread"); + finished_condition.notify_all(); + } + } + // info("worker ", worker_num, " done!"); + } + // info("worker exit ", worker_num); +} +} // namespace azteclabs::wsdb + +#endif diff --git a/native-packages/wsdb/cpp/src/common/thread_pool.hpp b/native-packages/wsdb/cpp/src/common/thread_pool.hpp new file mode 100644 index 000000000000..a753709fda60 --- /dev/null +++ b/native-packages/wsdb/cpp/src/common/thread_pool.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace azteclabs::wsdb { +class ThreadPool { + public: + ThreadPool(size_t num_threads); + ThreadPool(const ThreadPool& other) = delete; + ThreadPool(ThreadPool&& other) = delete; + ~ThreadPool(); + + ThreadPool& operator=(const ThreadPool& other) = delete; + ThreadPool& operator=(ThreadPool&& other) = delete; + + void enqueue(const std::function& task); + void wait(); + size_t num_threads() { return workers.size(); }; + + private: + std::vector workers; + std::queue> tasks; + std::mutex tasks_mutex; + std::condition_variable condition; + std::condition_variable finished_condition; + std::atomic tasks_running; + bool stop = false; + + void worker_loop(size_t thread_index); +}; +} // namespace azteclabs::wsdb diff --git a/native-packages/wsdb/cpp/src/common/uint256.hpp b/native-packages/wsdb/cpp/src/common/uint256.hpp new file mode 100644 index 000000000000..1b01888380ab --- /dev/null +++ b/native-packages/wsdb/cpp/src/common/uint256.hpp @@ -0,0 +1,88 @@ +#pragma once + +#include "field/field_element.hpp" +#include +#include + +// Minimal 256-bit unsigned integer for wsdb, replacing barretenberg/numeric/uint256. +// FieldElement (canonical bytes, no arithmetic) is the production key/value type; this +// type exists for the reference/test code (e.g. the nullifier memory tree's abs-diff +// closest-leaf search) that genuinely needs integer arithmetic on field values. +namespace azteclabs::wsdb::numeric { + +struct uint256_t { + uint64_t data[4]{}; // little-endian limbs + + uint256_t() = default; + uint256_t(uint64_t v) { data[0] = v; } + + // From a FieldElement (big-endian canonical 32 bytes) -> little-endian limbs. + uint256_t(const ::azteclabs::wsdb::FieldElement& fe) + { + for (int limb = 0; limb < 4; ++limb) { + uint64_t v = 0; + for (int b = 0; b < 8; ++b) { + v = (v << 8) | fe.bytes[static_cast((3 - limb) * 8 + b)]; + } + data[static_cast(limb)] = v; + } + } + + bool operator==(const uint256_t& o) const + { + return data[0] == o.data[0] && data[1] == o.data[1] && data[2] == o.data[2] && data[3] == o.data[3]; + } + + std::strong_ordering operator<=>(const uint256_t& o) const + { + for (int i = 3; i >= 0; --i) { + if (data[i] != o.data[i]) { + return data[i] <=> o.data[i]; + } + } + return std::strong_ordering::equal; + } + + uint256_t operator+(const uint256_t& o) const + { + uint256_t r; + unsigned __int128 carry = 0; + for (int i = 0; i < 4; ++i) { + unsigned __int128 cur = (unsigned __int128)data[i] + o.data[i] + carry; + r.data[i] = static_cast(cur); + carry = cur >> 64; + } + return r; + } + + uint256_t operator-(const uint256_t& o) const + { + uint256_t r; + unsigned __int128 borrow = 0; + for (int i = 0; i < 4; ++i) { + unsigned __int128 cur = (unsigned __int128)data[i] - o.data[i] - borrow; + r.data[i] = static_cast(cur); + borrow = (cur >> 64) & 1U; + } + return r; + } + + // Back to a FieldElement (little-endian limbs -> big-endian canonical bytes). + operator ::azteclabs::wsdb::FieldElement() const + { + ::azteclabs::wsdb::FieldElement fe; + for (int limb = 0; limb < 4; ++limb) { + uint64_t v = data[static_cast(limb)]; + for (int b = 0; b < 8; ++b) { + fe.bytes[static_cast((3 - limb) * 8 + b)] = static_cast(v >> (8 * (7 - b))); + } + } + return fe; + } +}; + +} // namespace azteclabs::wsdb::numeric + +namespace azteclabs::wsdb { +using numeric::uint256_t; +} // namespace azteclabs::wsdb diff --git a/native-packages/wsdb/cpp/src/common/uint256.test.cpp b/native-packages/wsdb/cpp/src/common/uint256.test.cpp new file mode 100644 index 000000000000..7cba00ee5342 --- /dev/null +++ b/native-packages/wsdb/cpp/src/common/uint256.test.cpp @@ -0,0 +1,47 @@ +#include "common/uint256.hpp" +#include "field/field_element.hpp" + +#include +#include +#include +#include + +// Backward-compatibility lock for the on-disk lmdb KEY format. wsdb keys nodes/leaves by +// FrKeyType = numeric::uint256_t, and lmdblib serialises a key by raw memcpy of the struct. +// barretenberg's uint256_t (and ours) is { uint64_t data[4] } with data[0] = least- +// significant limb, so the on-disk key is the canonical value in LITTLE-ENDIAN limb order. +// FieldElement stores big-endian canonical bytes; this verifies the BE<->LE conversion and +// the resulting key bytes match the pre-decouple format exactly. +using azteclabs::wsdb::numeric::uint256_t; +using azteclabs::wsdb::FieldElement; + +TEST(Uint256KeyFormat, LittleEndianCanonicalKeyBytes) +{ + // Canonical value 0x...0102030405060708 (low 64 bits set, rest zero). + FieldElement fe(std::string("0x0000000000000000000000000000000000000000000000000102030405060708")); + uint256_t k(fe); + EXPECT_EQ(k.data[0], 0x0102030405060708ULL); + EXPECT_EQ(k.data[1], 0ULL); + EXPECT_EQ(k.data[2], 0ULL); + EXPECT_EQ(k.data[3], 0ULL); + + // The lmdb key is a raw memcpy of the uint256_t -> little-endian limb bytes. + uint8_t buf[32]; + std::memcpy(buf, &k, 32); + uint8_t expected[32] = { 0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01 }; + EXPECT_EQ(0, std::memcmp(buf, expected, 32)); + + // And it round-trips back to the same field element. + EXPECT_EQ(static_cast(k), fe); +} + +TEST(Uint256KeyFormat, LimbExtractionAcrossFullWidth) +{ + FieldElement fe(std::string("0xfedcba9876543210ffffffffffffffff00000000000000001122334455667788")); + uint256_t k(fe); + EXPECT_EQ(k.data[3], 0xfedcba9876543210ULL); // most-significant limb + EXPECT_EQ(k.data[2], 0xffffffffffffffffULL); + EXPECT_EQ(k.data[1], 0x0000000000000000ULL); + EXPECT_EQ(k.data[0], 0x1122334455667788ULL); // least-significant limb + EXPECT_EQ(static_cast(k), fe); +} diff --git a/native-packages/wsdb/cpp/src/field/field_element.hpp b/native-packages/wsdb/cpp/src/field/field_element.hpp new file mode 100644 index 000000000000..c5db7cad5b9c --- /dev/null +++ b/native-packages/wsdb/cpp/src/field/field_element.hpp @@ -0,0 +1,132 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace azteclabs::wsdb { + +/** + * @brief A 256-bit field element as wsdb sees it: 32 canonical bytes, no arithmetic. + * + * wsdb never does field arithmetic — it only stores field values (node hashes, leaf + * keys), orders them, compares them, serialises them, and feeds them to the hash + * primitive. So it owns this minimal value type instead of depending on barretenberg's + * `fr` (and the montgomery form, constexpr field machinery, and stdlib coupling that + * pulling in `fr` implies). The 32 bytes are barretenberg's canonical field + * serialisation (big-endian): byte order == numeric order, and the msgpack encoding is + * byte-identical to `azteclabs::wsdb::fr` (a 32-byte bin), so on-disk data and the IPC wire are + * unchanged. + */ +struct FieldElement { + std::array bytes{}; + + FieldElement() = default; + explicit FieldElement(const std::array& b) + : bytes(b) + {} + + // Construct from a small integer (canonical big-endian), matching azteclabs::wsdb::fr(uint64_t). + FieldElement(uint64_t v) + { + for (size_t i = 0; i < sizeof(v); ++i) { + bytes[bytes.size() - 1 - i] = static_cast(v >> (8 * i)); + } + } + + // Construct from a "0x..."-style hex string (big-endian), matching azteclabs::wsdb::fr(std::string). + // Right-aligned: shorter strings are zero-padded on the left. Used by tests/fixtures. + explicit FieldElement(const std::string& hex) + { + std::string h = (hex.rfind("0x", 0) == 0 || hex.rfind("0X", 0) == 0) ? hex.substr(2) : hex; + if (h.size() > 64) { + h = h.substr(h.size() - 64); + } + h = std::string(64 - h.size(), '0') + h; + auto nib = [](char c) -> uint8_t { + if (c >= '0' && c <= '9') { + return static_cast(c - '0'); + } + return static_cast((c | 0x20) - 'a' + 10); + }; + for (size_t i = 0; i < 32; ++i) { + bytes[i] = static_cast((nib(h[2 * i]) << 4) | nib(h[2 * i + 1])); + } + } + + static FieldElement zero() { return {}; } + + // A random field element, guaranteed < modulus (top byte zero => < 2^248 < bn254 r). + // Test/fixture helper, mirroring azteclabs::wsdb::fr::random_element(). + static FieldElement random_element() + { + static thread_local std::mt19937_64 rng{ std::random_device{}() }; + FieldElement fe; + for (size_t i = 1; i < fe.bytes.size(); ++i) { + fe.bytes[i] = static_cast(rng()); + } + return fe; + } + [[nodiscard]] bool is_zero() const { return *this == FieldElement{}; } + + // Big-endian canonical bytes => lexicographic byte order is numeric order, which is + // what the indexed-tree low-leaf search and the lmdb key comparator require. + bool operator==(const FieldElement&) const = default; + auto operator<=>(const FieldElement&) const = default; + + [[nodiscard]] const uint8_t* data() const { return bytes.data(); } + uint8_t* data() { return bytes.data(); } + + // Raw 32-byte (de)serialisation, mirroring azteclabs::wsdb::fr's buffer API. + static void serialize_to_buffer(const FieldElement& v, uint8_t* buffer) { std::memcpy(buffer, v.bytes.data(), 32); } + static FieldElement serialize_from_buffer(const uint8_t* buffer) + { + FieldElement fe; + std::memcpy(fe.bytes.data(), buffer, 32); + return fe; + } + + // msgpack: a 32-byte bin, byte-identical to azteclabs::wsdb::fr. The bytes are already canonical + // (no montgomery form here), so we pack/unpack them directly. Intrusive members, + // matching azteclabs::wsdb::fr, so msgpack-c dispatches to them. + void msgpack_pack(auto& packer) const + { + packer.pack_bin(static_cast(bytes.size())); + packer.pack_bin_body(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + } + void msgpack_unpack(auto o) { bytes = static_cast>(o); } +}; + +inline std::ostream& operator<<(std::ostream& os, const FieldElement& fe) +{ + static constexpr char hex[] = "0123456789abcdef"; + os << "0x"; + for (uint8_t b : fe.bytes) { + os << hex[b >> 4] << hex[b & 0xf]; + } + return os; +} + +} // namespace azteclabs::wsdb + +// In the decoupled (bb-free) build there is no barretenberg `fr`; wsdb's merkle code +// refers to the node/leaf field type as `fr`/`azteclabs::wsdb::fr`, so alias it to FieldElement. +// Lets the forked merkle headers compile unchanged. The parity/equivalence tests DO link +// barretenberg and need the real azteclabs::wsdb::fr alongside FieldElement, so they compile with +// -DWSDB_NO_FR_ALIAS to suppress this alias. +namespace azteclabs::wsdb { +using fr = ::azteclabs::wsdb::FieldElement; +} // namespace azteclabs::wsdb + +template <> struct std::hash { + size_t operator()(const azteclabs::wsdb::FieldElement& fe) const noexcept + { + size_t h = 0; + std::memcpy(&h, fe.bytes.data(), sizeof(h)); + return h; + } +}; diff --git a/native-packages/wsdb/cpp/src/field/field_element.test.cpp b/native-packages/wsdb/cpp/src/field/field_element.test.cpp new file mode 100644 index 000000000000..f368ff813d01 --- /dev/null +++ b/native-packages/wsdb/cpp/src/field/field_element.test.cpp @@ -0,0 +1,101 @@ +#include "field/field_element.hpp" +#include "field/hash_policy.hpp" +#include "field/poseidon2.hpp" + +#include "barretenberg/aztec/aztec_hash_policy.hpp" +#include "barretenberg/crypto/merkle_tree/hash.hpp" +#include "barretenberg/ecc/curves/bn254/fr.hpp" +#include +#include + +// Consensus gate: the wsdb FieldElement + c_bind hashing path must produce +// byte-identical results to barretenberg's Poseidon2HashPolicy, which is what the +// AVM proves against and what is committed on L1. If this holds, replacing bb::fr +// with FieldElement and the hash policy with the c_bind cannot change any root. +namespace { +azteclabs::wsdb::FieldElement to_fe(const bb::fr& x) +{ + azteclabs::wsdb::FieldElement fe; + bb::fr::serialize_to_buffer(x, fe.data()); + return fe; +} +} // namespace + +TEST(FieldElementCbind, HashMatchesBbPolicy) +{ + for (int i = 0; i < 256; ++i) { + bb::fr a = bb::fr::random_element(); + bb::fr b = bb::fr::random_element(); + bb::fr expected = bb::crypto::merkle_tree::Poseidon2HashPolicy::hash_pair(a, b); + azteclabs::wsdb::FieldElement got = azteclabs::wsdb::poseidon2_hash({ to_fe(a), to_fe(b) }); + EXPECT_EQ(got, to_fe(expected)); + } +} + +TEST(FieldElementCbind, HashPairWithSeparatorMatchesBbPolicy) +{ + for (uint64_t sep : { uint64_t(0), uint64_t(1), uint64_t(7), uint64_t(1) << 40 }) { + for (int i = 0; i < 64; ++i) { + bb::fr a = bb::fr::random_element(); + bb::fr b = bb::fr::random_element(); + bb::fr expected = bb::crypto::merkle_tree::Poseidon2HashPolicy::hash_pair_with_separator(sep, a, b); + azteclabs::wsdb::FieldElement got = + azteclabs::wsdb::poseidon2_hash_pair_with_separator(sep, to_fe(a), to_fe(b)); + EXPECT_EQ(got, to_fe(expected)); + } + } +} + +TEST(FieldElementCbind, OrderingMatchesNumericOrder) +{ + // Big-endian canonical bytes => lexicographic FieldElement order == numeric fr order. + for (int i = 0; i < 256; ++i) { + bb::fr a = bb::fr::random_element(); + bb::fr b = bb::fr::random_element(); + bool fe_lt = to_fe(a) < to_fe(b); + bool num_lt = uint256_t(a) < uint256_t(b); + EXPECT_EQ(fe_lt, num_lt); + } +} + +// msgpack byte-identical to bb::fr => on-disk lmdb data and the IPC wire are unchanged. +TEST(FieldElementMsgpack, ByteIdenticalToBbFr) +{ + for (int i = 0; i < 256; ++i) { + bb::fr a = bb::fr::random_element(); + msgpack::sbuffer fe_buf; + msgpack::pack(fe_buf, to_fe(a)); + msgpack::sbuffer fr_buf; + msgpack::pack(fr_buf, a); + ASSERT_EQ(fe_buf.size(), fr_buf.size()); + EXPECT_EQ(0, std::memcmp(fe_buf.data(), fr_buf.data(), fe_buf.size())); + // round-trips back to the same value + azteclabs::wsdb::FieldElement rt; + msgpack::unpack(fe_buf.data(), fe_buf.size()).get().convert(rt); + EXPECT_EQ(rt, to_fe(a)); + } +} + +TEST(FieldElementMsgpack, FromUint64MatchesBbFr) +{ + for (uint64_t v : { uint64_t(0), uint64_t(1), uint64_t(900), (uint64_t(1) << 63) | 7 }) { + EXPECT_EQ(azteclabs::wsdb::FieldElement(v), to_fe(bb::fr(v))); + } +} + +// The domain-separated policies must match bb::aztec's exactly (consensus). +TEST(HashPolicy, DomainSeparatedPoliciesMatchBb) +{ + for (int i = 0; i < 64; ++i) { + bb::fr a = bb::fr::random_element(); + bb::fr b = bb::fr::random_element(); + auto fa = to_fe(a); + auto fb = to_fe(b); + EXPECT_EQ(azteclabs::wsdb::NullifierMerkleHashPolicy::hash_pair(fa, fb), + to_fe(bb::aztec::NullifierMerkleHashPolicy::hash_pair(a, b))); + EXPECT_EQ(azteclabs::wsdb::PublicDataMerkleHashPolicy::hash_pair(fa, fb), + to_fe(bb::aztec::PublicDataMerkleHashPolicy::hash_pair(a, b))); + EXPECT_EQ(azteclabs::wsdb::AztecMerkleHashPolicy::hash_pair(fa, fb), + to_fe(bb::aztec::AztecMerkleHashPolicy::hash_pair(a, b))); + } +} diff --git a/native-packages/wsdb/cpp/src/field/hash_policy.hpp b/native-packages/wsdb/cpp/src/field/hash_policy.hpp new file mode 100644 index 000000000000..1418e1817a3d --- /dev/null +++ b/native-packages/wsdb/cpp/src/field/hash_policy.hpp @@ -0,0 +1,62 @@ +#pragma once + +#include "common/aztec_constants.hpp" +#include "field/field_element.hpp" +#include "field/poseidon2.hpp" +#include +#include + +namespace azteclabs::wsdb { + +// Protocol domain separators - CONSENSUS constants sourced from the generated +// aztec_constants.hpp (single source of truth; the AVM hashes internal nodes with the same +// values). Per-tree separators stop sibling paths being transported across trees. +inline constexpr uint64_t DOM_SEP_MERKLE_HASH = DOM_SEP__MERKLE_HASH; +inline constexpr uint64_t DOM_SEP_NULLIFIER_MERKLE = DOM_SEP__NULLIFIER_MERKLE; +inline constexpr uint64_t DOM_SEP_PUBLIC_DATA_MERKLE = DOM_SEP__PUBLIC_DATA_MERKLE; +inline constexpr uint64_t DOM_SEP_WRITTEN_SLOTS_MERKLE = DOM_SEP__WRITTEN_SLOTS_MERKLE; +inline constexpr uint64_t DOM_SEP_RETRIEVED_BYTECODES_MERKLE = DOM_SEP__RETRIEVED_BYTECODES_MERKLE; + +// Baseline Poseidon2 policy (no domain separator), routed through the bb c_bind. +struct Poseidon2HashPolicy { + static FieldElement hash(const std::vector& inputs) { return poseidon2_hash(inputs); } + static FieldElement hash_pair(const FieldElement& lhs, const FieldElement& rhs) + { + return poseidon2_hash({ lhs, rhs }); + } + static FieldElement hash_pair_with_separator(uint64_t separator, const FieldElement& lhs, const FieldElement& rhs) + { + return poseidon2_hash_pair_with_separator(separator, lhs, rhs); + } + static FieldElement zero_hash() { return FieldElement::zero(); } +}; + +// Domain-separated internal-node hashing (mirrors azteclabs::wsdb::aztec::AztecMerkleHashPolicyT). +template struct AztecMerkleHashPolicyT : public Poseidon2HashPolicy { + static FieldElement hash_pair(const FieldElement& lhs, const FieldElement& rhs) + { + return Poseidon2HashPolicy::hash_pair_with_separator(Separator, lhs, rhs); + } +}; + +using NullifierMerkleHashPolicy = AztecMerkleHashPolicyT; +using PublicDataMerkleHashPolicy = AztecMerkleHashPolicyT; +using WrittenSlotsMerkleHashPolicy = AztecMerkleHashPolicyT; +using RetrievedBytecodesMerkleHashPolicy = AztecMerkleHashPolicyT; +// Append-only trees (note-hash, L1->L2 message, archive) share the baseline separator. +using AztecMerkleHashPolicy = AztecMerkleHashPolicyT; + +} // namespace azteclabs::wsdb + +// In the decoupled build, the forked merkle code refers to these under barretenberg's +// original namespaces (merkle_tree::Poseidon2HashPolicy, aztec::*). Inject them. +namespace azteclabs::wsdb::merkle_tree { +using ::azteclabs::wsdb::Poseidon2HashPolicy; +} // namespace azteclabs::wsdb::merkle_tree +namespace azteclabs::wsdb::aztec { +using ::azteclabs::wsdb::AztecMerkleHashPolicy; +using ::azteclabs::wsdb::NullifierMerkleHashPolicy; +using ::azteclabs::wsdb::PublicDataMerkleHashPolicy; +using ::azteclabs::wsdb::RetrievedBytecodesMerkleHashPolicy; +using ::azteclabs::wsdb::WrittenSlotsMerkleHashPolicy; +} // namespace azteclabs::wsdb::aztec diff --git a/native-packages/wsdb/cpp/src/field/merkle_hash_policy_for.hpp b/native-packages/wsdb/cpp/src/field/merkle_hash_policy_for.hpp new file mode 100644 index 000000000000..1cae0b9898c6 --- /dev/null +++ b/native-packages/wsdb/cpp/src/field/merkle_hash_policy_for.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include "field/field_element.hpp" +#include "field/hash_policy.hpp" +#include "merkle_tree/indexed_leaf.hpp" + +// Leaf-type -> per-tree hash policy trait. Split out of hash_policy.hpp because it depends +// on the leaf types (indexed_leaf.hpp); keeping it separate lets the bb-linked parity test +// use the policies without dragging in the forked merkle headers (which need the fr alias). +namespace azteclabs::wsdb { + +template struct MerkleHashPolicyFor; +template <> struct MerkleHashPolicyFor { + using type = NullifierMerkleHashPolicy; +}; +template <> struct MerkleHashPolicyFor { + using type = PublicDataMerkleHashPolicy; +}; +template <> struct MerkleHashPolicyFor { + using type = AztecMerkleHashPolicy; +}; +template using MerkleHashPolicyForT = typename MerkleHashPolicyFor::type; + +} // namespace azteclabs::wsdb + +namespace azteclabs::wsdb::aztec { +template using MerkleHashPolicyForT = ::azteclabs::wsdb::MerkleHashPolicyForT; +} // namespace azteclabs::wsdb::aztec diff --git a/native-packages/wsdb/cpp/src/field/poseidon2.hpp b/native-packages/wsdb/cpp/src/field/poseidon2.hpp new file mode 100644 index 000000000000..de11e7644312 --- /dev/null +++ b/native-packages/wsdb/cpp/src/field/poseidon2.hpp @@ -0,0 +1,39 @@ +#pragma once + +#include "field/field_element.hpp" +#include +#include +#include + +// Flat C ABI into barretenberg's Poseidon2 (defined in libbarretenberg.a, +// crypto/poseidon2/poseidon2_capi.cpp). Field elements cross as 32 canonical bytes. +// This is the only computation wsdb cannot own: the hash must match what the AVM +// proves and what is committed on L1, so it is single-sourced in barretenberg. +extern "C" { +void bb_poseidon2_hash(const uint8_t* elems, size_t n, uint8_t* out); +void bb_poseidon2_hash_pair_with_separator(uint64_t separator, const uint8_t* lhs, const uint8_t* rhs, uint8_t* out); +} + +namespace azteclabs::wsdb { + +inline FieldElement poseidon2_hash(const std::vector& inputs) +{ + std::vector buf(inputs.size() * 32); + for (size_t i = 0; i < inputs.size(); ++i) { + std::memcpy(buf.data() + (i * 32), inputs[i].data(), 32); + } + FieldElement out; + bb_poseidon2_hash(buf.data(), inputs.size(), out.data()); + return out; +} + +inline FieldElement poseidon2_hash_pair_with_separator(uint64_t separator, + const FieldElement& lhs, + const FieldElement& rhs) +{ + FieldElement out; + bb_poseidon2_hash_pair_with_separator(separator, lhs.data(), rhs.data(), out.data()); + return out; +} + +} // namespace azteclabs::wsdb diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp b/native-packages/wsdb/cpp/src/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp similarity index 98% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp index e119ae1606ad..8a96874e120e 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp @@ -8,31 +8,27 @@ #include #include -#include #include -#include #include #include -#include -#include #include #include #include #include -#include "barretenberg/common/thread_pool.hpp" -#include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" -#include "barretenberg/crypto/merkle_tree/response.hpp" -#include "barretenberg/crypto/merkle_tree/signal.hpp" -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/numeric/bitop/pow.hpp" +#include "common/bitop.hpp" +#include "common/thread_pool.hpp" +#include "merkle_tree/hash_path.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/response.hpp" +#include "merkle_tree/signal.hpp" +#include "merkle_tree/tree_db_stats.hpp" +#include "merkle_tree/tree_meta.hpp" +#include "merkle_tree/types.hpp" -namespace bb::crypto::merkle_tree { +namespace azteclabs::wsdb::merkle_tree { -using namespace bb; +using namespace azteclabs::wsdb; /** * @brief Implements a simple append-only merkle tree @@ -1256,4 +1252,4 @@ void ContentAddressedAppendOnlyTree::add_batch_internal( store_->put_meta(meta); } -} // namespace bb::crypto::merkle_tree +} // namespace azteclabs::wsdb::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp b/native-packages/wsdb/cpp/src/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp similarity index 98% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp rename to native-packages/wsdb/cpp/src/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp index ef5f713561b6..2c29b1a432c4 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp @@ -1,37 +1,32 @@ -#include "barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" -#include "../fixtures.hpp" -#include "../memory_tree.hpp" -#include "../test_fixtures.hpp" -#include "barretenberg/common/streams.hpp" -#include "barretenberg/common/test.hpp" -#include "barretenberg/common/thread_pool.hpp" -#include "barretenberg/crypto/merkle_tree/hash.hpp" -#include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/array_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/response.hpp" -#include "barretenberg/crypto/merkle_tree/signal.hpp" -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/relations/relation_parameters.hpp" +#include "merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" +#include "field/field_element.hpp" +#include "field/hash_policy.hpp" +#include "merkle_tree/fixtures.hpp" +#include "merkle_tree/hash_path.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/memory_tree.hpp" +#include "merkle_tree/node_store/cached_content_addressed_tree_store.hpp" +#include "merkle_tree/response.hpp" +#include "merkle_tree/signal.hpp" +#include "merkle_tree/test_fixtures.hpp" +#include "merkle_tree/tree_db_stats.hpp" +#include "merkle_tree/types.hpp" #include -#include #include #include #include #include +#include #include #include #include #include -using namespace bb; -using namespace bb::crypto::merkle_tree; -using namespace bb::lmdblib; +using namespace azteclabs::wsdb; +using namespace azteclabs::wsdb::merkle_tree; +using namespace azteclabs::lmdblib; -using Store = ContentAddressedCachedTreeStore; +using Store = ContentAddressedCachedTreeStore; using TreeType = ContentAddressedAppendOnlyTree; using LMDBStoreType = LMDBTreeStore; @@ -618,7 +613,7 @@ TEST_F(PersistedContentAddressedAppendOnlyTreeTest, can_commit_and_restore) check_root(tree, memdb.root()); check_sibling_path(tree, 0, memdb.get_sibling_path(0)); - bb::fr initial_root = memdb.root(); + azteclabs::wsdb::fr initial_root = memdb.root(); fr_sibling_path initial_sibling_path = memdb.get_sibling_path(0); memdb.update_element(0, get_value(0)); add_value(tree, get_value(0)); diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.hpp b/native-packages/wsdb/cpp/src/merkle_tree/fixtures.hpp similarity index 84% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/fixtures.hpp index b29045166ef2..19bce2f91adb 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.hpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/fixtures.hpp @@ -6,12 +6,14 @@ #include #include -#include "barretenberg/common/thread_pool.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/numeric/random/engine.hpp" +#include "common/log.hpp" +#include "common/random.hpp" +#include "common/thread_pool.hpp" +#include "field/field_element.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/tree_db_stats.hpp" -namespace bb::crypto::merkle_tree { +namespace azteclabs::wsdb::merkle_tree { const uint32_t NUM_VALUES = 1024; inline auto& engine = numeric::get_debug_randomness(); @@ -71,4 +73,4 @@ void inline print_store_data(LMDBTreeStore::SharedPtr db, std::ostream& os) os << stats; } -} // namespace bb::crypto::merkle_tree +} // namespace azteclabs::wsdb::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.test.cpp b/native-packages/wsdb/cpp/src/merkle_tree/fixtures.test.cpp similarity index 62% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.test.cpp rename to native-packages/wsdb/cpp/src/merkle_tree/fixtures.test.cpp index cf12fa1a4bdd..2adfbecc61ff 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.test.cpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/fixtures.test.cpp @@ -1,6 +1,6 @@ #include "fixtures.hpp" -namespace bb::crypto::merkle_tree { +namespace azteclabs::wsdb::merkle_tree { const fr& get_value(size_t index) { @@ -8,4 +8,4 @@ const fr& get_value(size_t index) return VALUES[index]; } -} // namespace bb::crypto::merkle_tree +} // namespace azteclabs::wsdb::merkle_tree diff --git a/native-packages/wsdb/cpp/src/merkle_tree/hash_path.hpp b/native-packages/wsdb/cpp/src/merkle_tree/hash_path.hpp new file mode 100644 index 000000000000..80af92a60e5f --- /dev/null +++ b/native-packages/wsdb/cpp/src/merkle_tree/hash_path.hpp @@ -0,0 +1,30 @@ +// === AUDIT STATUS === +// internal: { status: Complete, auditors: [Nishat], commit: 22d6fc368da0fbe5412f4f7b2890a052aa48d803 } +// external_1: { status: not started, auditors: [], commit: } +// external_2: { status: not started, auditors: [], commit: } +// ===================== + +#pragma once +#include "field/field_element.hpp" +#include + +namespace azteclabs::wsdb::merkle_tree { + +using fr_hash_path = std::vector>; +using fr_sibling_path = std::vector; + +} // namespace azteclabs::wsdb::merkle_tree + +// We add to std namespace as fr_hash_path is actually a std::vector, and this is the only way +// to achieve effective ADL. +namespace std { +inline std::ostream& operator<<(std::ostream& os, azteclabs::wsdb::merkle_tree::fr_hash_path const& path) +{ + os << "[\n"; + for (size_t i = 0; i < path.size(); ++i) { + os << " (" << i << ": " << path[i].first << ", " << path[i].second << ")\n"; + } + os << "]\n"; + return os; +} +} // namespace std diff --git a/native-packages/wsdb/cpp/src/merkle_tree/indexed_leaf.hpp b/native-packages/wsdb/cpp/src/merkle_tree/indexed_leaf.hpp new file mode 100644 index 000000000000..90fd9404b511 --- /dev/null +++ b/native-packages/wsdb/cpp/src/merkle_tree/indexed_leaf.hpp @@ -0,0 +1,201 @@ +// === AUDIT STATUS === +// internal: { status: Complete, auditors: [Nishat], commit: 22d6fc368da0fbe5412f4f7b2890a052aa48d803 } +// external_1: { status: not started, auditors: [], commit: } +// external_2: { status: not started, auditors: [], commit: } +// ===================== + +#pragma once + +#include "field/field_element.hpp" +#include "merkle_tree/types.hpp" +#include + +namespace azteclabs::wsdb::merkle_tree { + +struct NullifierLeafValue { + fr nullifier; + + MSGPACK_DEFINE_MAP(nullifier) + + NullifierLeafValue() = default; + NullifierLeafValue(const fr& n) + : nullifier(n) + {} + NullifierLeafValue(const NullifierLeafValue& other) = default; + NullifierLeafValue(NullifierLeafValue&& other) = default; + NullifierLeafValue& operator=(const NullifierLeafValue& other) + { + if (this != &other) { + nullifier = other.nullifier; + } + return *this; + } + + NullifierLeafValue& operator=(NullifierLeafValue&& other) noexcept + { + if (this != &other) { + nullifier = other.nullifier; + } + return *this; + } + ~NullifierLeafValue() = default; + + static bool is_updateable() { return false; } + + bool operator==(NullifierLeafValue const& other) const { return nullifier == other.nullifier; } + + friend std::ostream& operator<<(std::ostream& os, const NullifierLeafValue& v) + { + os << "nullifier = " << v.nullifier; + return os; + } + + fr get_key() const { return nullifier; } + + bool is_empty() const { return nullifier.is_zero(); } + + std::vector get_hash_inputs(fr nextKey, fr nextIndex) const + { + return std::vector({ nullifier, nextKey, nextIndex }); + } + + operator uint256_t() const { return get_key(); } + + static NullifierLeafValue empty() { return { fr::zero() }; } + + static NullifierLeafValue padding(index_t i) { return { i }; } + + size_t hash() const noexcept { return std::hash{}(nullifier); } + + static std::string name() { return "NullifierLeafValue"; }; +}; + +struct PublicDataLeafValue { + fr slot; + fr value; + + MSGPACK_DEFINE_MAP(slot, value) + + PublicDataLeafValue() = default; + PublicDataLeafValue(const fr& s, const fr& v) + : slot(s) + , value(v) + {} + PublicDataLeafValue(const PublicDataLeafValue& other) = default; + PublicDataLeafValue(PublicDataLeafValue&& other) = default; + PublicDataLeafValue& operator=(const PublicDataLeafValue& other) + { + if (this != &other) { + value = other.value; + slot = other.slot; + } + return *this; + } + + PublicDataLeafValue& operator=(PublicDataLeafValue&& other) noexcept + { + if (this != &other) { + value = other.value; + slot = other.slot; + } + return *this; + } + ~PublicDataLeafValue() = default; + + static bool is_updateable() { return true; } + + bool operator==(PublicDataLeafValue const& other) const { return value == other.value && slot == other.slot; } + + friend std::ostream& operator<<(std::ostream& os, const PublicDataLeafValue& v) + { + os << "slot = " << v.slot << " : value = " << v.value; + return os; + } + + fr get_key() const { return slot; } + + bool is_empty() const { return slot == fr::zero() && value == fr::zero(); } + + std::vector get_hash_inputs(fr nextSlot, fr nextIndex) const + { + return std::vector({ slot, value, nextSlot, nextIndex }); + } + + operator uint256_t() const { return get_key(); } + + static PublicDataLeafValue empty() { return { fr::zero(), fr::zero() }; } + + static PublicDataLeafValue padding(index_t i) { return { i, fr::zero() }; } + + size_t hash() const noexcept { return std::hash{}(value) ^ (std::hash{}(slot) << 1); } + + static std::string name() { return "PublicDataLeafValue"; }; +}; + +template struct IndexedLeaf { + LeafType leaf; + index_t nextIndex; + fr nextKey; + + MSGPACK_DEFINE_MAP(leaf, nextIndex, nextKey) + + IndexedLeaf() = default; + + IndexedLeaf(const LeafType& leaf, const index_t& nextIdx, const fr& nextKey) + : leaf(leaf) + , nextIndex(nextIdx) + , nextKey(nextKey) + {} + + IndexedLeaf(const IndexedLeaf& other) = default; + IndexedLeaf(IndexedLeaf&& other) noexcept = default; + ~IndexedLeaf() = default; + + static bool is_updateable() { return LeafType::is_updateable(); } + + static std::string name() { return LeafType::name(); } + + bool operator==(IndexedLeaf const& other) const + { + return leaf == other.leaf && nextKey == other.nextKey && nextIndex == other.nextIndex; + } + + IndexedLeaf& operator=(IndexedLeaf const& other) + { + if (this != &other) { + leaf = other.leaf; + nextKey = other.nextKey; + nextIndex = other.nextIndex; + } + return *this; + } + + IndexedLeaf& operator=(IndexedLeaf&& other) noexcept + { + if (this != &other) { + leaf = other.leaf; + nextKey = other.nextKey; + nextIndex = other.nextIndex; + } + return *this; + } + + friend std::ostream& operator<<(std::ostream& os, const IndexedLeaf& leaf) + { + os << leaf.leaf << "\nnextIdx = " << leaf.nextIndex << "\nnextKey = " << leaf.nextKey; + return os; + } + + std::vector get_hash_inputs() const { return leaf.get_hash_inputs(nextKey, nextIndex); } + + // A leaf hashes to zero only when it is the true padding leaf: an empty value with null next pointers. + // The next-pointer check keeps a real low leaf whose value happens to be zero (e.g. the head of the list) + // from being mistaken for padding. + bool is_empty() const { return leaf.is_empty() && nextKey.is_zero() && nextIndex == 0; } + + static IndexedLeaf empty() { return { LeafType::empty(), 0, 0 }; } + + static IndexedLeaf padding(index_t i) { return { LeafType::padding(i), 0, 0 }; } +}; + +} // namespace azteclabs::wsdb::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp b/native-packages/wsdb/cpp/src/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp similarity index 98% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp index 5874c88ec48d..f3dce156448c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp @@ -12,34 +12,26 @@ #include #include #include -#include #include #include #include -#include #include #include #include #include #include -#include "barretenberg/common/assert.hpp" -#include "barretenberg/common/thread_pool.hpp" -#include "barretenberg/common/utils.hpp" -#include "barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" -#include "barretenberg/crypto/merkle_tree/hash.hpp" -#include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" -#include "barretenberg/crypto/merkle_tree/response.hpp" -#include "barretenberg/crypto/merkle_tree/signal.hpp" -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/numeric/bitop/get_msb.hpp" -#include "barretenberg/numeric/uint256/uint256.hpp" - -namespace bb::crypto::merkle_tree { +#include "common/bitop.hpp" +#include "common/thread_pool.hpp" +#include "merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" +#include "merkle_tree/hash_path.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/response.hpp" +#include "merkle_tree/signal.hpp" +#include "merkle_tree/tree_meta.hpp" +#include "merkle_tree/types.hpp" + +namespace azteclabs::wsdb::merkle_tree { template struct ContentAddressedIndexedTreeTestAccess; @@ -315,7 +307,7 @@ ContentAddressedIndexedTree::ContentAddressedIndexedTree( } std::vector appended_leaves; - std::vector appended_hashes; + std::vector appended_hashes; std::vector initial_set; auto num_default_values = static_cast(initial_size - prefilled_values.size()); for (uint32_t i = 0; i < num_default_values; ++i) { @@ -1575,4 +1567,4 @@ void ContentAddressedIndexedTree::generate_sequential_inse completion); } -} // namespace bb::crypto::merkle_tree +} // namespace azteclabs::wsdb::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp b/native-packages/wsdb/cpp/src/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp similarity index 99% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp rename to native-packages/wsdb/cpp/src/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp index 902924f5cef5..d887f647fdf2 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp @@ -1,31 +1,29 @@ #include "./content_addressed_indexed_tree.hpp" -#include "../fixtures.hpp" -#include "../hash.hpp" -#include "../node_store/array_store.hpp" -#include "../nullifier_tree/nullifier_memory_tree.hpp" -#include "../test_fixtures.hpp" -#include "barretenberg/common/streams.hpp" -#include "barretenberg/common/test.hpp" -#include "barretenberg/common/thread_pool.hpp" -#include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/response.hpp" -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/numeric/random/engine.hpp" +#include "common/random.hpp" +#include "field/hash_policy.hpp" +#include "merkle_tree/fixtures.hpp" +#include "merkle_tree/hash_path.hpp" +#include "merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/node_store/cached_content_addressed_tree_store.hpp" +#include "merkle_tree/nullifier_tree/nullifier_memory_tree.hpp" +#include "merkle_tree/response.hpp" +#include "merkle_tree/test_fixtures.hpp" +#include "merkle_tree/tree_db_stats.hpp" +#include "merkle_tree/types.hpp" #include #include #include #include #include #include +#include #include #include #include #include -namespace bb::crypto::merkle_tree { +namespace azteclabs::wsdb::merkle_tree { template struct ContentAddressedIndexedTreeTestAccess { using Tree = ContentAddressedIndexedTree; using LeafUpdate = typename Tree::LeafUpdate; @@ -40,10 +38,10 @@ template struct ContentAddressedIndexed tree.perform_updates_without_witness(highest_index, std::move(updates), completion); } }; -} // namespace bb::crypto::merkle_tree +} // namespace azteclabs::wsdb::merkle_tree -using namespace bb; -using namespace bb::crypto::merkle_tree; +using namespace azteclabs::wsdb; +using namespace azteclabs::wsdb::merkle_tree; using HashPolicy = Poseidon2HashPolicy; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp b/native-packages/wsdb/cpp/src/merkle_tree/lmdb_store/lmdb_tree_store.cpp similarity index 93% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp rename to native-packages/wsdb/cpp/src/merkle_tree/lmdb_store/lmdb_tree_store.cpp index 0ce2bf5b47d0..a5fc61def492 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/lmdb_store/lmdb_tree_store.cpp @@ -4,29 +4,23 @@ // external_2: { status: not started, auditors: [], commit: } // ===================== -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/common/serialize.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/lmdblib/lmdb_db_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/lmdblib/lmdb_store_base.hpp" -#include "barretenberg/numeric/uint128/uint128.hpp" -#include "barretenberg/numeric/uint256/uint256.hpp" -#include "barretenberg/serialize/msgpack_impl.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "field/field_element.hpp" #include "lmdb_tree_store.hpp" +#include "lmdblib/lmdb_db_transaction.hpp" +#include "lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_store_base.hpp" +#include "merkle_tree/tree_db_stats.hpp" +#include "merkle_tree/types.hpp" #include #include #include -#include #include #include #include -#include #include -namespace bb::crypto::merkle_tree { +namespace azteclabs::wsdb::merkle_tree { /** * Integer key comparison function. @@ -282,7 +276,11 @@ void LMDBTreeStore::decrement_node_reference_count(const fr& nodeHash, NodePaylo } if (--nodeData.ref == 0) { // std::cout << "Deleting node at " << nodeHash << std::endl; - tx.delete_value(nodeHash, *_nodeDatabase); + // Wrap in FrKeyType so the node key is serialised identically to write_node/read_node + // (canonical uint256 bytes). Passing the raw fr would bind serialise_key's generic + // template and emit Montgomery-form bytes, deleting the wrong node. + FrKeyType key(nodeHash); + tx.delete_value(key, *_nodeDatabase); return; } // std::cout << "Updating node at " << nodeHash << " ref is now " << nodeData.ref << std::endl; @@ -334,4 +332,4 @@ void LMDBTreeStore::write_node(const fr& nodeHash, const NodePayload& nodeData, tx.put_value(key, encoded, *_nodeDatabase); } -} // namespace bb::crypto::merkle_tree +} // namespace azteclabs::wsdb::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp b/native-packages/wsdb/cpp/src/merkle_tree/lmdb_store/lmdb_tree_store.hpp similarity index 90% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/lmdb_store/lmdb_tree_store.hpp index c916924f4e69..56363d8677e4 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/lmdb_store/lmdb_tree_store.hpp @@ -5,32 +5,24 @@ // ===================== #pragma once -#include "barretenberg/common/log.hpp" -#include "barretenberg/common/serialize.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/lmdblib/lmdb_database.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_read_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_store_base.hpp" -#include "barretenberg/lmdblib/lmdb_write_transaction.hpp" -#include "barretenberg/serialize/msgpack_impl.hpp" -#include "barretenberg/world_state/types.hpp" -#include "lmdb.h" +#include "field/field_element.hpp" +#include "lmdblib/lmdb_database.hpp" +#include "lmdblib/lmdb_read_transaction.hpp" +#include "lmdblib/lmdb_store_base.hpp" +#include "lmdblib/lmdb_write_transaction.hpp" +#include "merkle_tree/tree_db_stats.hpp" +#include "merkle_tree/tree_meta.hpp" +#include "merkle_tree/types.hpp" #include +#include #include #include #include #include -#include -#include -#include -namespace bb::crypto::merkle_tree { +namespace azteclabs::wsdb::merkle_tree { -using namespace bb::lmdblib; +using namespace azteclabs::lmdblib; struct BlockPayload { @@ -38,7 +30,7 @@ struct BlockPayload { block_number_t blockNumber; fr root; - SERIALIZATION_FIELDS(size, blockNumber, root) + MSGPACK_DEFINE_MAP(size, blockNumber, root) bool operator==(const BlockPayload& other) const { @@ -58,7 +50,7 @@ struct NodePayload { std::optional right; uint64_t ref; - SERIALIZATION_FIELDS(left, right, ref) + MSGPACK_DEFINE_MAP(left, right, ref) bool operator==(const NodePayload& other) const { @@ -69,7 +61,7 @@ struct NodePayload { struct BlockIndexPayload { std::vector blockNumbers; - SERIALIZATION_FIELDS(blockNumbers) + MSGPACK_DEFINE_MAP(blockNumbers) bool operator==(const BlockIndexPayload& other) const { return blockNumbers == other.blockNumbers; } @@ -263,4 +255,4 @@ template bool LMDBTreeStore::get_node_data(const fr& nodeHash, } return success; } -} // namespace bb::crypto::merkle_tree +} // namespace azteclabs::wsdb::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp b/native-packages/wsdb/cpp/src/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp similarity index 96% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp rename to native-packages/wsdb/cpp/src/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp index cbad34090212..e453bf0f0392 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp @@ -8,27 +8,18 @@ #include #include -#include "barretenberg/common/serialize.hpp" -#include "barretenberg/common/streams.hpp" -#include "barretenberg/common/test.hpp" -#include "barretenberg/crypto/merkle_tree/fixtures.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/numeric/random/engine.hpp" -#include "barretenberg/numeric/uint128/uint128.hpp" -#include "barretenberg/numeric/uint256/uint256.hpp" -#include "barretenberg/stdlib/primitives/field/field.hpp" +#include +#include +#include "merkle_tree/fixtures.hpp" +#include "merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/tree_meta.hpp" +#include "merkle_tree/types.hpp" +#include "lmdblib/lmdb_helpers.hpp" #include "lmdb_tree_store.hpp" -using namespace bb::stdlib; -using namespace bb::crypto::merkle_tree; +using namespace azteclabs::wsdb::merkle_tree; -using Builder = bb::UltraCircuitBuilder; -using field_ct = field_t; -using witness_ct = witness_t; class LMDBTreeStoreTest : public testing::Test { protected: @@ -229,7 +220,7 @@ TEST_F(LMDBTreeStoreTest, can_serde_64bit_values) TEST_F(LMDBTreeStoreTest, can_write_and_read_leaf_indices) { index_t index = 47; - bb::fr key = get_value(5); + azteclabs::wsdb::fr key = get_value(5); LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); { LMDBWriteTransaction::Ptr transaction = store.create_write_transaction(); @@ -255,7 +246,7 @@ TEST_F(LMDBTreeStoreTest, can_write_and_read_nodes) nodePayload.left = get_value(4); nodePayload.right = get_value(5); nodePayload.ref = 4; - bb::fr key = get_value(6); + azteclabs::wsdb::fr key = get_value(6); LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); { LMDBWriteTransaction::Ptr transaction = store.create_write_transaction(); @@ -280,7 +271,7 @@ TEST_F(LMDBTreeStoreTest, can_write_and_read_leaves_by_hash) PublicDataLeafValue leafData; leafData.slot = get_value(0); leafData.value = get_value(1); - bb::fr key = get_value(2); + azteclabs::wsdb::fr key = get_value(2); LMDBTreeStore store(_directory, "DB1", _mapSize, _maxReaders); { LMDBWriteTransaction::Ptr transaction = store.create_write_transaction(); diff --git a/native-packages/wsdb/cpp/src/merkle_tree/memory_tree.hpp b/native-packages/wsdb/cpp/src/merkle_tree/memory_tree.hpp new file mode 100644 index 000000000000..e7ea607700eb --- /dev/null +++ b/native-packages/wsdb/cpp/src/merkle_tree/memory_tree.hpp @@ -0,0 +1,136 @@ +// === AUDIT STATUS === +// internal: { status: Complete, auditors: [Nishat], commit: 22d6fc368da0fbe5412f4f7b2890a052aa48d803 } +// external_1: { status: not started, auditors: [], commit: } +// external_2: { status: not started, auditors: [], commit: } +// ===================== + +#pragma once +#include "common/assert.hpp" +#include "hash_path.hpp" + +namespace azteclabs::wsdb::merkle_tree { + +/** + * A MemoryTree is structured as follows: + * hashes_ + * +------------------------------------------------------------------------------+ + * | 0 -> h_{0,0} h_{0,1} h_{0,2} h_{0,3} h_{0,4} h_{0,5} h_{0,6} h_{0,7} | + * i | | + * n | 8 -> h_{1,0} h_{1,1} h_{1,2} h_{1,3} | + * d | | + * e | 12 -> h_{2,0} h_{2,1} | + * x | | + * | 14 -> h_{3,0} | + * +------------------------------------------------------------------------------+ + * + * Here, depth_ = 3 and {h_{0,j}}_{i=0..7} are leaf values. + * Also, root_ = h_{3,0} and total_size_ = (2 * 8 - 2) = 14. + * Lastly, h_{i,j} = hash( h_{i-1,2j}, h_{i-1,2j+1} ) where i > 1. + */ +template class MemoryTree { + public: + MemoryTree(size_t depth); + + fr_hash_path get_hash_path(size_t index) const; + + fr_sibling_path get_sibling_path(size_t index) const; + + fr update_element(size_t index, fr const& value); + + fr root() const { return root_; } + + fr get_node(uint32_t level, size_t index) const; + + public: + size_t depth_; + size_t total_size_; + azteclabs::wsdb::fr root_; + std::vector hashes_; +}; + +template +MemoryTree::MemoryTree(size_t depth) + : depth_(depth) +{ + + BB_ASSERT_GTE(depth_, 1U); + BB_ASSERT_LTE(depth, 20U); + total_size_ = 1UL << depth_; + hashes_.resize(total_size_ * 2 - 2); + + // Build the entire tree. + auto current = fr(0); + size_t layer_size = total_size_; + for (size_t offset = 0; offset < hashes_.size(); offset += layer_size, layer_size /= 2) { + for (size_t i = 0; i < layer_size; ++i) { + hashes_[offset + i] = current; + } + current = HashingPolicy::hash_pair(current, current); + } + + root_ = current; +} + +template fr MemoryTree::get_node(uint32_t level, size_t index) const +{ + auto offset = 0UL; + for (size_t i = depth_; i > level; i--) { + offset += 1UL << i; + } + return hashes_[offset + index]; +} + +template fr_hash_path MemoryTree::get_hash_path(size_t index) const +{ + fr_hash_path path(depth_); + size_t offset = 0; + size_t layer_size = total_size_; + for (size_t i = 0; i < depth_; ++i) { + index -= index & 0x1; + path[i] = std::make_pair(hashes_[offset + index], hashes_[offset + index + 1]); + offset += layer_size; + layer_size >>= 1; + index >>= 1; + } + return path; +} + +template fr_sibling_path MemoryTree::get_sibling_path(size_t index) const +{ + fr_sibling_path path(depth_); + size_t offset = 0; + size_t layer_size = total_size_; + for (size_t i = 0; i < depth_; i++) { + if (index % 2 == 0) { + path[i] = hashes_[offset + index + 1]; + } else { + path[i] = hashes_[offset + index - 1]; + } + offset += layer_size; + layer_size >>= 1; + index >>= 1; + } + return path; +} + +template fr MemoryTree::update_element(size_t index, fr const& value) +{ + if (index >= total_size_) { + throw std::runtime_error("update_element: index out of range"); + } + size_t offset = 0; + size_t layer_size = total_size_; + fr current = value; + for (size_t i = 0; i < depth_; ++i) { + hashes_[offset + index] = current; + index &= (~0ULL) - 1; + current = HashingPolicy::hash_pair(hashes_[offset + index], hashes_[offset + index + 1]); + offset += layer_size; + layer_size >>= 1; + index >>= 1; + } + root_ = current; + return root_; +} + +} // namespace azteclabs::wsdb::merkle_tree diff --git a/native-packages/wsdb/cpp/src/merkle_tree/merkle_tree_id.cpp b/native-packages/wsdb/cpp/src/merkle_tree/merkle_tree_id.cpp new file mode 100644 index 000000000000..66866d564d41 --- /dev/null +++ b/native-packages/wsdb/cpp/src/merkle_tree/merkle_tree_id.cpp @@ -0,0 +1,26 @@ +#include "merkle_tree/merkle_tree_id.hpp" + +#include +#include + +namespace azteclabs::wsdb::merkle_tree { + +std::string getMerkleTreeName(MerkleTreeId id) +{ + switch (id) { + case MerkleTreeId::NULLIFIER_TREE: + return "NullifierTree"; + case MerkleTreeId::NOTE_HASH_TREE: + return "NoteHashTree"; + case MerkleTreeId::PUBLIC_DATA_TREE: + return "PublicDataTree"; + case MerkleTreeId::L1_TO_L2_MESSAGE_TREE: + return "L1ToL2MessageTree"; + case MerkleTreeId::ARCHIVE: + return "ArchiveTree"; + default: + throw std::invalid_argument("Unknown MerkleTreeId"); + } +} + +} // namespace azteclabs::wsdb::merkle_tree diff --git a/native-packages/wsdb/cpp/src/merkle_tree/merkle_tree_id.hpp b/native-packages/wsdb/cpp/src/merkle_tree/merkle_tree_id.hpp new file mode 100644 index 000000000000..a322938781f7 --- /dev/null +++ b/native-packages/wsdb/cpp/src/merkle_tree/merkle_tree_id.hpp @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include + +#include "merkle_tree/types.hpp" +#include + +namespace azteclabs::wsdb::merkle_tree { + +enum MerkleTreeId { + NULLIFIER_TREE = 0, + NOTE_HASH_TREE = 1, + PUBLIC_DATA_TREE = 2, + L1_TO_L2_MESSAGE_TREE = 3, + ARCHIVE = 4, +}; + +std::string getMerkleTreeName(MerkleTreeId id); + +/** + * @brief Identifies a (possibly forked, possibly historical) view of the merkle trees. + * + * This is the lightweight handle exchanged across the merkle-DB IPC boundary; it carries no + * persistent-store dependency, which is why it lives in crypto/merkle_tree rather than world_state. + */ +struct WorldStateRevision { + // Sentinel value for `blockNumber` indicating "not pinned to any historical block; + // use the latest committed state of the underlying tree". This is distinct from + // `blockNumber == 0`, which means "pin to block 0 (the initial / genesis state)". + // We use the maximum uint32_t rather than 0 because 0 is a valid historical block + // number (the genesis header), and overloading 0 caused silent regressions where + // genesis-anchored witnesses would return the current tip instead of genesis. + static constexpr block_number_t LATEST = std::numeric_limits::max(); + + index_t forkId{ 0 }; + block_number_t blockNumber{ LATEST }; + bool includeUncommitted{ false }; + + MSGPACK_DEFINE_MAP(forkId, blockNumber, includeUncommitted) + + static WorldStateRevision committed() { return WorldStateRevision{ .includeUncommitted = false }; } + static WorldStateRevision uncommitted() { return WorldStateRevision{ .includeUncommitted = true }; } + + // True when the revision is pinned to a specific historical block rather than the latest state. + bool is_historical() const { return blockNumber != LATEST; } + + bool operator==(const WorldStateRevision& other) const = default; +}; + +} // namespace azteclabs::wsdb::merkle_tree + +MSGPACK_ADD_ENUM(azteclabs::wsdb::merkle_tree::MerkleTreeId) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/array_store.hpp b/native-packages/wsdb/cpp/src/merkle_tree/node_store/array_store.hpp similarity index 90% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/array_store.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/node_store/array_store.hpp index a04918875c16..e8d6973dc269 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/array_store.hpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/node_store/array_store.hpp @@ -5,12 +5,11 @@ // ===================== #pragma once -#include "./tree_meta.hpp" -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/stdlib/primitives/field/field.hpp" +#include "merkle_tree/tree_meta.hpp" +#include "merkle_tree/types.hpp" #include -namespace bb::crypto::merkle_tree { +namespace azteclabs::wsdb::merkle_tree { class MockTransaction { public: @@ -76,13 +75,13 @@ template class ArrayStore { } return slot.first; } - void put_meta(const index_t& size, const bb::fr& root) + void put_meta(const index_t& size, const azteclabs::wsdb::fr& root) { meta.root = root; meta.size = size; } - void get_meta(index_t& size, bb::fr& root, ReadTransaction&, bool) const + void get_meta(index_t& size, azteclabs::wsdb::fr& root, ReadTransaction&, bool) const { size = meta.size; root = meta.root; @@ -97,4 +96,4 @@ template class ArrayStore { std::vector>>> map; TreeMeta meta; }; -} // namespace bb::crypto::merkle_tree +} // namespace azteclabs::wsdb::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp b/native-packages/wsdb/cpp/src/merkle_tree/node_store/cached_content_addressed_tree_store.hpp similarity index 98% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/node_store/cached_content_addressed_tree_store.hpp index 0c075e838fb1..94eb1f4e1285 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/node_store/cached_content_addressed_tree_store.hpp @@ -5,32 +5,24 @@ // ===================== #pragma once -#include "./tree_meta.hpp" -#include "barretenberg/common/log.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/content_addressed_cache.hpp" -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/lmdblib/lmdb_transaction.hpp" -#include "barretenberg/numeric/uint256/uint256.hpp" -#include "barretenberg/serialize/msgpack.hpp" -#include "barretenberg/stdlib/primitives/field/field.hpp" -#include "msgpack/assert.hpp" +#include "field/field_element.hpp" +#include "lmdblib/lmdb_helpers.hpp" +#include "merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/node_store/content_addressed_cache.hpp" +#include "merkle_tree/tree_db_stats.hpp" +#include "merkle_tree/tree_meta.hpp" +#include "merkle_tree/types.hpp" #include #include -#include #include #include #include -#include #include -#include #include #include -namespace bb::crypto::merkle_tree { +namespace azteclabs::wsdb::merkle_tree { /** * @brief Serves as a key-value node store for merkle trees. Caches all changes in memory before persisting them during @@ -1302,4 +1294,4 @@ void ContentAddressedCachedTreeStore::initialize_from_block(const } } -} // namespace bb::crypto::merkle_tree +} // namespace azteclabs::wsdb::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/content_addressed_cache.hpp b/native-packages/wsdb/cpp/src/merkle_tree/node_store/content_addressed_cache.hpp similarity index 96% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/content_addressed_cache.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/node_store/content_addressed_cache.hpp index 40ef06125ed4..298b60301757 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/content_addressed_cache.hpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/node_store/content_addressed_cache.hpp @@ -5,28 +5,20 @@ // ===================== #pragma once -#include "./tree_meta.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/numeric/uint256/uint256.hpp" -#include "barretenberg/serialize/msgpack.hpp" -#include "barretenberg/stdlib/primitives/field/field.hpp" -#include "msgpack/assert.hpp" +#include "merkle_tree/tree_meta.hpp" +#include "merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/types.hpp" +#include "field/field_element.hpp" #include -#include -#include #include -#include #include -#include #include #include #include #include -namespace bb::crypto::merkle_tree { +namespace azteclabs::wsdb::merkle_tree { // Stores all of the penidng updates to a mekle tree indexed for optimal retrieval // Also stores a journal of inverse changes to the cache, enabling checkpoints and @@ -495,4 +487,4 @@ void ContentAddressedCache::put_node_by_index(uint32_t level, con } nodes_by_index_[level][index] = node; } -} // namespace bb::crypto::merkle_tree +} // namespace azteclabs::wsdb::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/content_addressed_cache.test.cpp b/native-packages/wsdb/cpp/src/merkle_tree/node_store/content_addressed_cache.test.cpp similarity index 98% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/content_addressed_cache.test.cpp rename to native-packages/wsdb/cpp/src/merkle_tree/node_store/content_addressed_cache.test.cpp index e690308530a0..7d22e41b9422 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/content_addressed_cache.test.cpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/node_store/content_addressed_cache.test.cpp @@ -1,14 +1,14 @@ -#include "barretenberg/crypto/merkle_tree/node_store/content_addressed_cache.hpp" -#include "barretenberg/common/test.hpp" -#include "barretenberg/crypto/merkle_tree/fixtures.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" -#include "barretenberg/ecc/curves/bn254/fr.hpp" +#include "merkle_tree/node_store/content_addressed_cache.hpp" +#include +#include "merkle_tree/fixtures.hpp" +#include "merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/tree_meta.hpp" +#include "field/field_element.hpp" #include #include -using namespace bb; -using namespace bb::crypto::merkle_tree; +using namespace azteclabs::wsdb; +using namespace azteclabs::wsdb::merkle_tree; using LeafValueType = PublicDataLeafValue; using IndexedLeafType = IndexedLeaf; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp b/native-packages/wsdb/cpp/src/merkle_tree/nullifier_tree/nullifier_leaf.hpp similarity index 88% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/nullifier_tree/nullifier_leaf.hpp index de4180b00e1b..d98fc7f9b1cf 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/nullifier_tree/nullifier_leaf.hpp @@ -5,12 +5,10 @@ // ===================== #pragma once -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/crypto/pedersen_commitment/pedersen.hpp" -#include "barretenberg/serialize/msgpack.hpp" -#include "barretenberg/stdlib/hash/poseidon2/poseidon2.hpp" +#include "merkle_tree/types.hpp" +#include -namespace bb::crypto::merkle_tree { +namespace azteclabs::wsdb::merkle_tree { struct indexed_nullifier_leaf { fr value; @@ -18,7 +16,7 @@ struct indexed_nullifier_leaf { fr nextValue; // For serialization, update with any new fields - SERIALIZATION_FIELDS(value, nextIndex, nextValue); + MSGPACK_DEFINE_MAP(value, nextIndex, nextValue); bool operator==(indexed_nullifier_leaf const&) const = default; std::ostream& operator<<(std::ostream& os) @@ -78,11 +76,11 @@ template class WrappedNullifierLeaf { /** * @brief Return the hash of the wrapped object, other return the zero hash of 0 * - * @return bb::fr + * @return azteclabs::wsdb::fr */ - bb::fr hash() const + azteclabs::wsdb::fr hash() const { - return data.has_value() ? HashingPolicy::hash(data.value().get_hash_inputs()) : bb::fr::zero(); + return data.has_value() ? HashingPolicy::hash(data.value().get_hash_inputs()) : azteclabs::wsdb::fr::zero(); } /** @@ -126,4 +124,4 @@ inline std::pair find_closest_leaf(std::vector(it - diff.begin()), repeated); } -} // namespace bb::crypto::merkle_tree +} // namespace azteclabs::wsdb::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp b/native-packages/wsdb/cpp/src/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp similarity index 96% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp index b13f0f7500d8..8d84f6d8776d 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp @@ -5,13 +5,12 @@ // ===================== #pragma once -#include "../hash.hpp" -#include "../memory_tree.hpp" -#include "barretenberg/common/assert.hpp" -#include "barretenberg/crypto/merkle_tree/hash_path.hpp" +#include "common/assert.hpp" +#include "merkle_tree/hash_path.hpp" +#include "merkle_tree/memory_tree.hpp" #include "nullifier_leaf.hpp" -namespace bb::crypto::merkle_tree { +namespace azteclabs::wsdb::merkle_tree { /** * An NullifierMemoryTree is structured just like a usual merkle tree: @@ -83,7 +82,7 @@ template class NullifierMemoryTree : public MemoryTree< fr_sibling_path update_element(fr const& value); - const std::vector& get_hashes() { return hashes_; } + const std::vector& get_hashes() { return hashes_; } const WrappedNullifierLeaf get_leaf(size_t index) { return (index < leaves_.size()) ? leaves_[index] @@ -190,4 +189,4 @@ template fr_sibling_path NullifierMemoryTree -using namespace bb; -using namespace bb::crypto::merkle_tree; +using namespace azteclabs::wsdb; +using namespace azteclabs::wsdb::merkle_tree; using HashPolicy = Poseidon2HashPolicy; using WrappedLeaf = WrappedNullifierLeaf; diff --git a/native-packages/wsdb/cpp/src/merkle_tree/response.hpp b/native-packages/wsdb/cpp/src/merkle_tree/response.hpp new file mode 100644 index 000000000000..6db7554cb03d --- /dev/null +++ b/native-packages/wsdb/cpp/src/merkle_tree/response.hpp @@ -0,0 +1,276 @@ +// === AUDIT STATUS === +// internal: { status: Complete, auditors: [Nishat], commit: 22d6fc368da0fbe5412f4f7b2890a052aa48d803 } +// external_1: { status: not started, auditors: [], commit: } +// external_2: { status: not started, auditors: [], commit: } +// ===================== + +#pragma once + +#include "field/field_element.hpp" +#include "merkle_tree/hash_path.hpp" +#include "merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/tree_meta.hpp" +#include "merkle_tree/types.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace azteclabs::wsdb::merkle_tree { +struct TreeMetaResponse { + TreeMeta meta; + + TreeMetaResponse() = default; + ~TreeMetaResponse() = default; + TreeMetaResponse(const TreeMetaResponse& other) = default; + TreeMetaResponse(TreeMetaResponse&& other) noexcept = default; + TreeMetaResponse& operator=(const TreeMetaResponse& other) = default; + TreeMetaResponse& operator=(TreeMetaResponse&& other) noexcept = default; +}; + +struct CheckpointResponse { + uint32_t depth; + + CheckpointResponse() = default; + ~CheckpointResponse() = default; + CheckpointResponse(const CheckpointResponse& other) = default; + CheckpointResponse(CheckpointResponse&& other) noexcept = default; + CheckpointResponse& operator=(const CheckpointResponse& other) = default; + CheckpointResponse& operator=(CheckpointResponse&& other) noexcept = default; +}; + +struct AddDataResponse { + index_t size; + fr root; + + AddDataResponse() = default; + ~AddDataResponse() = default; + AddDataResponse(const AddDataResponse& other) = default; + AddDataResponse(AddDataResponse&& other) noexcept = default; + AddDataResponse& operator=(const AddDataResponse& other) = default; + AddDataResponse& operator=(AddDataResponse&& other) noexcept = default; +}; + +struct GetSiblingPathResponse { + fr_sibling_path path; + + GetSiblingPathResponse() = default; + ~GetSiblingPathResponse() = default; + GetSiblingPathResponse(const GetSiblingPathResponse& other) = default; + GetSiblingPathResponse(GetSiblingPathResponse&& other) noexcept = default; + GetSiblingPathResponse& operator=(const GetSiblingPathResponse& other) = default; + GetSiblingPathResponse& operator=(GetSiblingPathResponse&& other) noexcept = default; +}; + +template struct LeafUpdateWitnessData { + IndexedLeaf leaf; + index_t index; + fr_sibling_path path; + + LeafUpdateWitnessData(const IndexedLeaf& l, const index_t& i, fr_sibling_path p) + : leaf(l) + , index(i) + , path(std::move(p)) + {} + LeafUpdateWitnessData() = default; + ~LeafUpdateWitnessData() = default; + LeafUpdateWitnessData(const LeafUpdateWitnessData& other) = default; + LeafUpdateWitnessData(LeafUpdateWitnessData&& other) noexcept = default; + LeafUpdateWitnessData& operator=(const LeafUpdateWitnessData& other) = default; + LeafUpdateWitnessData& operator=(LeafUpdateWitnessData&& other) noexcept = default; + bool operator==(const LeafUpdateWitnessData& other) const = default; +}; + +template struct BatchInsertionResult { + std::vector> low_leaf_witness_data; + std::vector> sorted_leaves; + fr_sibling_path subtree_path; +}; + +template struct SequentialInsertionResult { + std::vector> low_leaf_witness_data; + std::vector> insertion_witness_data; +}; + +template struct AddIndexedDataResponse { + AddDataResponse add_data_result; + fr_sibling_path subtree_path; + std::shared_ptr>> sorted_leaves; + std::shared_ptr>> low_leaf_witness_data; + + AddIndexedDataResponse() = default; + ~AddIndexedDataResponse() = default; + AddIndexedDataResponse(const AddIndexedDataResponse& other) = default; + AddIndexedDataResponse(AddIndexedDataResponse&& other) noexcept = default; + AddIndexedDataResponse& operator=(const AddIndexedDataResponse& other) = default; + AddIndexedDataResponse& operator=(AddIndexedDataResponse&& other) noexcept = default; +}; + +template struct AddIndexedDataSequentiallyResponse { + AddDataResponse add_data_result; + std::shared_ptr>> low_leaf_witness_data; + std::shared_ptr>> insertion_witness_data; + + AddIndexedDataSequentiallyResponse() = default; + ~AddIndexedDataSequentiallyResponse() = default; + AddIndexedDataSequentiallyResponse(const AddIndexedDataSequentiallyResponse& other) = default; + AddIndexedDataSequentiallyResponse(AddIndexedDataSequentiallyResponse&& other) noexcept = default; + AddIndexedDataSequentiallyResponse& operator=(const AddIndexedDataSequentiallyResponse& other) = default; + AddIndexedDataSequentiallyResponse& operator=(AddIndexedDataSequentiallyResponse&& other) noexcept = default; +}; + +struct BlockForIndexResponse { + std::vector> blockNumbers; + + BlockForIndexResponse() = default; + ~BlockForIndexResponse() = default; + BlockForIndexResponse(const BlockForIndexResponse& other) = default; + BlockForIndexResponse(BlockForIndexResponse&& other) noexcept = default; + BlockForIndexResponse& operator=(const BlockForIndexResponse& other) = default; + BlockForIndexResponse& operator=(BlockForIndexResponse&& other) noexcept = default; +}; + +struct FindLeafIndexResponse { + std::vector> leaf_indices; + + FindLeafIndexResponse() = default; + ~FindLeafIndexResponse() = default; + FindLeafIndexResponse(const FindLeafIndexResponse& other) = default; + FindLeafIndexResponse(FindLeafIndexResponse&& other) noexcept = default; + FindLeafIndexResponse& operator=(const FindLeafIndexResponse& other) = default; + FindLeafIndexResponse& operator=(FindLeafIndexResponse&& other) noexcept = default; +}; + +struct SiblingPathAndIndex { + index_t index; + fr_sibling_path path; + + SiblingPathAndIndex() = default; + ~SiblingPathAndIndex() = default; + SiblingPathAndIndex(index_t index, fr_sibling_path path) + : index(index) + , path(path) + {} + SiblingPathAndIndex(const SiblingPathAndIndex& other) = default; + SiblingPathAndIndex(SiblingPathAndIndex&& other) noexcept = default; + SiblingPathAndIndex& operator=(const SiblingPathAndIndex& other) = default; + SiblingPathAndIndex& operator=(SiblingPathAndIndex&& other) noexcept = default; +}; + +struct FindLeafPathResponse { + std::vector> leaf_paths; + + FindLeafPathResponse() = default; + ~FindLeafPathResponse() = default; + FindLeafPathResponse(const FindLeafPathResponse& other) = default; + FindLeafPathResponse(FindLeafPathResponse&& other) noexcept = default; + FindLeafPathResponse& operator=(const FindLeafPathResponse& other) = default; + FindLeafPathResponse& operator=(FindLeafPathResponse&& other) noexcept = default; +}; + +struct GetLeafResponse { + std::optional leaf; + + GetLeafResponse() = default; + ~GetLeafResponse() = default; + GetLeafResponse(const GetLeafResponse& other) = default; + GetLeafResponse(GetLeafResponse&& other) noexcept = default; + GetLeafResponse& operator=(const GetLeafResponse& other) = default; + GetLeafResponse& operator=(GetLeafResponse&& other) noexcept = default; +}; + +template struct GetIndexedLeafResponse { + std::optional> indexed_leaf; +}; + +struct GetLowIndexedLeafResponse { + bool is_already_present; + index_t index; + + GetLowIndexedLeafResponse(bool p, const index_t& i) + : is_already_present(p) + , index(i) + {} + GetLowIndexedLeafResponse() = default; + ~GetLowIndexedLeafResponse() = default; + GetLowIndexedLeafResponse(const GetLowIndexedLeafResponse& other) = default; + GetLowIndexedLeafResponse(GetLowIndexedLeafResponse&& other) noexcept = default; + GetLowIndexedLeafResponse& operator=(const GetLowIndexedLeafResponse& other) = default; + GetLowIndexedLeafResponse& operator=(GetLowIndexedLeafResponse&& other) noexcept = default; + + bool operator==(const GetLowIndexedLeafResponse& other) const + { + return is_already_present == other.is_already_present && index == other.index; + } +}; + +template struct TypedResponse { + ResponseType inner; + bool success{ true }; + std::string message; + + TypedResponse() = default; + ~TypedResponse() = default; + TypedResponse(const TypedResponse& other) = default; + TypedResponse(TypedResponse&& other) noexcept = default; + TypedResponse& operator=(const TypedResponse& other) = default; + TypedResponse& operator=(TypedResponse&& other) noexcept = default; +}; + +struct Response { + bool success; + std::string message; + + Response(bool s, std::string m) + : success(s) + , message(std::move(m)) + {} + Response() = default; + ~Response() = default; + Response(const Response& other) = default; + Response(Response&& other) noexcept = default; + Response& operator=(const Response& other) = default; + Response& operator=(Response&& other) noexcept = default; +}; + +template +void execute_and_report(const std::function&)>& f, + const std::function&)>& on_completion) +{ + TypedResponse response; + try { + f(response); + } catch (std::exception& e) { + response.success = false; + response.message = e.what(); + // std::cout << "Response " << e.what() << std::endl; + } + try { + on_completion(response); + } catch (std::exception& e) { + std::cerr << "Completion callback threw: " << e.what() << std::endl; + std::abort(); + } +} + +inline void execute_and_report(const std::function& f, const std::function& on_completion) +{ + Response response{ true, "" }; + try { + f(); + } catch (std::exception& e) { + response.success = false; + response.message = e.what(); + // std::cout << "Response " << e.what() << std::endl; + } + try { + on_completion(response); + } catch (std::exception& e) { + std::cerr << "Completion callback threw: " << e.what() << std::endl; + std::abort(); + } +} +} // namespace azteclabs::wsdb::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/signal.hpp b/native-packages/wsdb/cpp/src/merkle_tree/signal.hpp similarity index 95% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/signal.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/signal.hpp index ac055746fc9d..7750b19f4ea9 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/signal.hpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/signal.hpp @@ -8,7 +8,7 @@ #include #include -namespace bb::crypto::merkle_tree { +namespace azteclabs::wsdb::merkle_tree { /** * @brief Used in parallel insertions in the the IndexedTree. Workers signal to other following workes as they move up * the level of the tree. @@ -66,4 +66,4 @@ class Signal { private: std::atomic signal_; }; -} // namespace bb::crypto::merkle_tree +} // namespace azteclabs::wsdb::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp b/native-packages/wsdb/cpp/src/merkle_tree/test_fixtures.hpp similarity index 96% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/test_fixtures.hpp index e7a56a52848f..5a60bd2f9975 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/test_fixtures.hpp @@ -1,16 +1,15 @@ #pragma once -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/response.hpp" -#include "barretenberg/crypto/merkle_tree/signal.hpp" -#include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/numeric/random/engine.hpp" +#include "merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/response.hpp" +#include "merkle_tree/signal.hpp" +#include "field/field_element.hpp" #include #include #include -namespace bb::crypto::merkle_tree { +namespace azteclabs::wsdb::merkle_tree { inline void check_block_and_root_data(LMDBTreeStore::SharedPtr db, block_number_t blockNumber, @@ -308,4 +307,4 @@ template void revert_tree_to_depth(TreeType& tree, uint32_t auto completion = [&](auto completion) { tree.revert_to_depth(depth, completion); }; call_operation(completion, expected_success); } -} // namespace bb::crypto::merkle_tree +} // namespace azteclabs::wsdb::merkle_tree diff --git a/native-packages/wsdb/cpp/src/merkle_tree/tree_db_stats.hpp b/native-packages/wsdb/cpp/src/merkle_tree/tree_db_stats.hpp new file mode 100644 index 000000000000..285d4bad4fea --- /dev/null +++ b/native-packages/wsdb/cpp/src/merkle_tree/tree_db_stats.hpp @@ -0,0 +1,130 @@ +#pragma once + +// Persistent-tree storage statistics and the merkle responses that carry them. +// These live in the wsdb package (not barretenberg's generic merkle core) +// because only the persistent, lmdb-backed trees report stats — the in-memory +// merkle tree has none. DBStats itself is owned by lmdblib (it is what lmdb +// reports). + +#include "lmdblib/types.hpp" +#include "merkle_tree/tree_meta.hpp" +#include +#include + +namespace azteclabs::wsdb::merkle_tree { + +using DBStats = azteclabs::lmdblib::DBStats; + +struct TreeDBStats { + uint64_t mapSize; + uint64_t physicalFileSize; + DBStats blocksDBStats; + DBStats nodesDBStats; + DBStats leafPreimagesDBStats; + DBStats leafIndicesDBStats; + DBStats blockIndicesDBStats; + + TreeDBStats() = default; + TreeDBStats(uint64_t mapSize, uint64_t physicalFileSize) + : mapSize(mapSize) + , physicalFileSize(physicalFileSize) + {} + TreeDBStats(uint64_t mapSize, + uint64_t physicalFileSize, + const DBStats& blockStats, + const DBStats& nodesStats, + const DBStats& leafPreimagesDBStats, + const DBStats& leafIndicesStats, + const DBStats& blockIndicesStats) + : mapSize(mapSize) + , physicalFileSize(physicalFileSize) + , blocksDBStats(blockStats) + , nodesDBStats(nodesStats) + , leafPreimagesDBStats(leafPreimagesDBStats) + , leafIndicesDBStats(leafIndicesStats) + , blockIndicesDBStats(blockIndicesStats) + {} + TreeDBStats(const TreeDBStats& other) = default; + TreeDBStats(TreeDBStats&& other) noexcept { *this = std::move(other); } + + ~TreeDBStats() = default; + + MSGPACK_DEFINE_MAP(mapSize, + physicalFileSize, + blocksDBStats, + nodesDBStats, + leafPreimagesDBStats, + leafIndicesDBStats, + blockIndicesDBStats) + + bool operator==(const TreeDBStats& other) const + { + return mapSize == other.mapSize && physicalFileSize == other.physicalFileSize && + blocksDBStats == other.blocksDBStats && nodesDBStats == other.nodesDBStats && + leafPreimagesDBStats == other.leafPreimagesDBStats && leafIndicesDBStats == other.leafIndicesDBStats && + blockIndicesDBStats == other.blockIndicesDBStats; + } + + TreeDBStats& operator=(TreeDBStats&& other) noexcept + { + if (this != &other) { + mapSize = other.mapSize; + physicalFileSize = other.physicalFileSize; + blocksDBStats = std::move(other.blocksDBStats); + nodesDBStats = std::move(other.nodesDBStats); + leafPreimagesDBStats = std::move(other.leafPreimagesDBStats); + leafIndicesDBStats = std::move(other.leafIndicesDBStats); + blockIndicesDBStats = std::move(other.blockIndicesDBStats); + } + return *this; + } + + TreeDBStats& operator=(const TreeDBStats& other) = default; + + friend std::ostream& operator<<(std::ostream& os, const TreeDBStats& stats) + { + os << "Map Size: " << stats.mapSize << ", Physical File Size: " << stats.physicalFileSize << " Blocks DB " + << stats.blocksDBStats << ", Nodes DB " << stats.nodesDBStats << ", Leaf Pre-images DB " + << stats.leafPreimagesDBStats << ", Leaf Indices DB " << stats.leafIndicesDBStats << ", Block Indices DB " + << stats.blockIndicesDBStats; + return os; + } +}; + +struct CommitResponse { + TreeMeta meta; + TreeDBStats stats; + + CommitResponse() = default; + ~CommitResponse() = default; + CommitResponse(const CommitResponse& other) = default; + CommitResponse(CommitResponse&& other) noexcept = default; + CommitResponse& operator=(const CommitResponse& other) = default; + CommitResponse& operator=(CommitResponse&& other) noexcept = default; +}; + +struct UnwindResponse { + TreeMeta meta; + TreeDBStats stats; + + UnwindResponse() = default; + ~UnwindResponse() = default; + UnwindResponse(const UnwindResponse& other) = default; + UnwindResponse(UnwindResponse&& other) noexcept = default; + UnwindResponse& operator=(const UnwindResponse& other) = default; + UnwindResponse& operator=(UnwindResponse&& other) noexcept = default; +}; + +struct RemoveHistoricResponse { + TreeMeta meta; + TreeDBStats stats; + + RemoveHistoricResponse() = default; + ~RemoveHistoricResponse() = default; + RemoveHistoricResponse(const RemoveHistoricResponse& other) = default; + RemoveHistoricResponse(RemoveHistoricResponse&& other) noexcept = default; + RemoveHistoricResponse& operator=(const RemoveHistoricResponse& other) = default; + RemoveHistoricResponse& operator=(RemoveHistoricResponse&& other) noexcept = default; +}; + +} // namespace azteclabs::wsdb::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp b/native-packages/wsdb/cpp/src/merkle_tree/tree_meta.hpp similarity index 78% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/tree_meta.hpp index 3127c46808bc..7d15bf580a44 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/tree_meta.hpp @@ -5,46 +5,46 @@ // ===================== #pragma once -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/serialize/msgpack.hpp" +#include "field/field_element.hpp" +#include "merkle_tree/types.hpp" #include +#include #include #include #include -namespace bb::crypto::merkle_tree { +namespace azteclabs::wsdb::merkle_tree { struct TreeMeta { std::string name; uint32_t depth; index_t size; index_t committedSize; - bb::fr root; + azteclabs::wsdb::fr root; index_t initialSize; - bb::fr initialRoot; + azteclabs::wsdb::fr initialRoot; block_number_t oldestHistoricBlock; block_number_t unfinalizedBlockHeight; block_number_t finalizedBlockHeight; - SERIALIZATION_FIELDS(name, - depth, - size, - committedSize, - root, - initialSize, - initialRoot, - oldestHistoricBlock, - unfinalizedBlockHeight, - finalizedBlockHeight) + MSGPACK_DEFINE_MAP(name, + depth, + size, + committedSize, + root, + initialSize, + initialRoot, + oldestHistoricBlock, + unfinalizedBlockHeight, + finalizedBlockHeight) TreeMeta(std::string n, uint32_t d, const index_t& s, const index_t& c, - const bb::fr& r, + const azteclabs::wsdb::fr& r, const index_t& is, - const bb::fr& ir, + const azteclabs::wsdb::fr& ir, const block_number_t& o, const block_number_t& u, const block_number_t& f) @@ -85,4 +85,4 @@ inline std::ostream& operator<<(std::ostream& os, const TreeMeta& meta) return os; } -} // namespace bb::crypto::merkle_tree +} // namespace azteclabs::wsdb::merkle_tree diff --git a/native-packages/wsdb/cpp/src/merkle_tree/types.hpp b/native-packages/wsdb/cpp/src/merkle_tree/types.hpp new file mode 100644 index 000000000000..6015f6443155 --- /dev/null +++ b/native-packages/wsdb/cpp/src/merkle_tree/types.hpp @@ -0,0 +1,65 @@ +// === AUDIT STATUS === +// internal: { status: Complete, auditors: [Nishat], commit: 22d6fc368da0fbe5412f4f7b2890a052aa48d803 } +// external_1: { status: not started, auditors: [], commit: } +// external_2: { status: not started, auditors: [], commit: } +// ===================== + +#pragma once + +#include "common/uint256.hpp" +#include "field/field_element.hpp" +#include +#include +namespace azteclabs::wsdb::merkle_tree { + +using index_t = uint64_t; +using block_number_t = uint32_t; +using LeafIndexKeyType = uint64_t; +using BlockMetaKeyType = uint64_t; +using FrKeyType = numeric::uint256_t; +using MetaKeyType = uint8_t; + +struct RequestContext { + bool includeUncommitted; + std::optional blockNumber; + azteclabs::wsdb::fr root; + std::optional maxIndex; +}; + +template fr preimage_to_key(const LeafType& leaf) +{ + return leaf.get_key(); +} + +inline fr preimage_to_key(const fr& leaf) +{ + return leaf; +} + +template bool is_empty(const LeafType& leaf) +{ + return leaf.is_empty(); +} + +inline bool is_empty(const fr& leaf) +{ + return leaf == fr::zero(); +} + +template constexpr bool requires_preimage_for_key() +{ + return true; +} + +template <> constexpr bool requires_preimage_for_key() +{ + return false; +} + +const std::string BLOCKS_DB = "blocks"; +const std::string NODES_DB = "nodes"; +const std::string LEAF_PREIMAGES_DB = "leaf preimages"; +const std::string LEAF_INDICES_DB = "leaf indices"; +const std::string BLOCK_INDICES_DB = "block indices"; + +} // namespace azteclabs::wsdb::merkle_tree diff --git a/native-packages/wsdb/cpp/src/world_state/fork.hpp b/native-packages/wsdb/cpp/src/world_state/fork.hpp new file mode 100644 index 000000000000..8c385643c3a5 --- /dev/null +++ b/native-packages/wsdb/cpp/src/world_state/fork.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include "field/hash_policy.hpp" +#include "merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" +#include "merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp" +#include "merkle_tree/node_store/cached_content_addressed_tree_store.hpp" +#include "world_state/tree_with_store.hpp" +#include "world_state/types.hpp" +#include +#include + +namespace azteclabs::wsdb::world_state { + +// Append-only trees (note-hash, L1->L2 message, archive) share the baseline merkle separator. +using AppendOnlyHashPolicy = aztec::AztecMerkleHashPolicy; + +using FrStore = merkle_tree::ContentAddressedCachedTreeStore; +using FrTree = merkle_tree::ContentAddressedAppendOnlyTree; + +using NullifierStore = merkle_tree::ContentAddressedCachedTreeStore; +using NullifierTree = + merkle_tree::ContentAddressedIndexedTree; + +using PublicDataStore = merkle_tree::ContentAddressedCachedTreeStore; +using PublicDataTree = + merkle_tree::ContentAddressedIndexedTree; + +using Tree = std::variant, TreeWithStore, TreeWithStore>; + +struct Fork { + using Id = uint64_t; + using SharedPtr = std::shared_ptr; + Id _forkId; + std::unordered_map _trees; + index_t _blockNumber; +}; +} // namespace azteclabs::wsdb::world_state diff --git a/barretenberg/cpp/src/barretenberg/world_state/tree_with_store.hpp b/native-packages/wsdb/cpp/src/world_state/tree_with_store.hpp similarity index 86% rename from barretenberg/cpp/src/barretenberg/world_state/tree_with_store.hpp rename to native-packages/wsdb/cpp/src/world_state/tree_with_store.hpp index d27630e1fe2b..d0b2382cfd03 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/tree_with_store.hpp +++ b/native-packages/wsdb/cpp/src/world_state/tree_with_store.hpp @@ -2,7 +2,7 @@ #include -namespace bb::world_state { +namespace azteclabs::wsdb::world_state { template struct TreeWithStore { using TreeType = Tree; @@ -23,4 +23,4 @@ template struct TreeWithStore { TreeWithStore& operator=(const TreeWithStore& other) = delete; }; -} // namespace bb::world_state +} // namespace azteclabs::wsdb::world_state diff --git a/native-packages/wsdb/cpp/src/world_state/types.cpp b/native-packages/wsdb/cpp/src/world_state/types.cpp new file mode 100644 index 000000000000..bddf49de4060 --- /dev/null +++ b/native-packages/wsdb/cpp/src/world_state/types.cpp @@ -0,0 +1,4 @@ +#include "world_state/types.hpp" + +// getMerkleTreeName now lives in crypto/merkle_tree/merkle_tree_id.cpp and is re-exported into +// azteclabs::wsdb::world_state via the `using namespace azteclabs::wsdb::merkle_tree` in types.hpp. diff --git a/barretenberg/cpp/src/barretenberg/world_state/types.hpp b/native-packages/wsdb/cpp/src/world_state/types.hpp similarity index 89% rename from barretenberg/cpp/src/barretenberg/world_state/types.hpp rename to native-packages/wsdb/cpp/src/world_state/types.hpp index 2aa4688e6af0..c64466cb268d 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/types.hpp +++ b/native-packages/wsdb/cpp/src/world_state/types.hpp @@ -5,27 +5,28 @@ #include #include -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/merkle_tree_id.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/serialize/msgpack.hpp" +#include "merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/tree_db_stats.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/merkle_tree_id.hpp" +#include "merkle_tree/tree_meta.hpp" +#include "merkle_tree/types.hpp" +#include "field/field_element.hpp" +#include -namespace bb::world_state { +namespace azteclabs::wsdb::world_state { -using namespace bb::crypto::merkle_tree; +using namespace azteclabs::wsdb::merkle_tree; // MerkleTreeId, getMerkleTreeName and WorldStateRevision are merkle-tree vocabulary that lives in // crypto/merkle_tree so that consumers (e.g. the AVM) can use them without depending on world_state. -// They are pulled in here via the `using namespace bb::crypto::merkle_tree` above and re-exported for +// They are pulled in here via the `using namespace azteclabs::wsdb::merkle_tree` above and re-exported for // the (many) existing world_state:: references. const uint64_t CANONICAL_FORK_ID = 0; const uint64_t NUM_TREES = 5; -using TreeStateReference = std::pair; +using TreeStateReference = std::pair; using StateReference = std::unordered_map; struct WorldStateStatusSummary { @@ -33,7 +34,7 @@ struct WorldStateStatusSummary { index_t finalizedBlockNumber; index_t oldestHistoricalBlock; bool treesAreSynched; - SERIALIZATION_FIELDS(unfinalizedBlockNumber, finalizedBlockNumber, oldestHistoricalBlock, treesAreSynched); + MSGPACK_DEFINE_MAP(unfinalizedBlockNumber, finalizedBlockNumber, oldestHistoricalBlock, treesAreSynched); WorldStateStatusSummary() = default; WorldStateStatusSummary(const index_t& unfinalizedBlockNumber, @@ -84,7 +85,7 @@ struct WorldStateDBStats { TreeDBStats publicDataTreeStats; TreeDBStats nullifierTreeStats; - SERIALIZATION_FIELDS( + MSGPACK_DEFINE_MAP( noteHashTreeStats, messageTreeStats, archiveTreeStats, publicDataTreeStats, nullifierTreeStats); WorldStateDBStats() = default; @@ -141,7 +142,7 @@ struct WorldStateMeta { TreeMeta publicDataTreeMeta; TreeMeta nullifierTreeMeta; - SERIALIZATION_FIELDS(noteHashTreeMeta, messageTreeMeta, archiveTreeMeta, publicDataTreeMeta, nullifierTreeMeta); + MSGPACK_DEFINE_MAP(noteHashTreeMeta, messageTreeMeta, archiveTreeMeta, publicDataTreeMeta, nullifierTreeMeta); WorldStateMeta() = default; WorldStateMeta(const TreeMeta& noteHashMeta, @@ -195,7 +196,7 @@ struct WorldStateStatusFull { WorldStateDBStats dbStats; WorldStateMeta meta; - SERIALIZATION_FIELDS(summary, dbStats, meta); + MSGPACK_DEFINE_MAP(summary, dbStats, meta); WorldStateStatusFull() = default; WorldStateStatusFull(const WorldStateStatusSummary& summary, @@ -233,4 +234,4 @@ struct WorldStateStatusFull { return os; } }; -} // namespace bb::world_state +} // namespace azteclabs::wsdb::world_state diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp b/native-packages/wsdb/cpp/src/world_state/world_state.cpp similarity index 95% rename from barretenberg/cpp/src/barretenberg/world_state/world_state.cpp rename to native-packages/wsdb/cpp/src/world_state/world_state.cpp index e2076c3144c5..8bdf7c5cbb73 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp +++ b/native-packages/wsdb/cpp/src/world_state/world_state.cpp @@ -1,19 +1,19 @@ -#include "barretenberg/world_state/world_state.hpp" -#include "barretenberg/aztec/aztec_constants.hpp" -#include "barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" -#include "barretenberg/crypto/merkle_tree/hash.hpp" -#include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" -#include "barretenberg/crypto/merkle_tree/response.hpp" -#include "barretenberg/crypto/merkle_tree/signal.hpp" -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/world_state/fork.hpp" -#include "barretenberg/world_state/tree_with_store.hpp" -#include "barretenberg/world_state/types.hpp" -#include "barretenberg/world_state/world_state_stores.hpp" +#include "world_state/world_state.hpp" +#include "common/aztec_constants.hpp" +#include "field/hash_policy.hpp" +#include "lmdblib/lmdb_helpers.hpp" +#include "merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" +#include "merkle_tree/hash_path.hpp" +#include "merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/response.hpp" +#include "merkle_tree/signal.hpp" +#include "merkle_tree/tree_meta.hpp" +#include "merkle_tree/types.hpp" +#include "world_state/fork.hpp" +#include "world_state/tree_with_store.hpp" +#include "world_state/types.hpp" +#include "world_state/world_state_stores.hpp" #include #include #include @@ -29,9 +29,9 @@ #include #include -namespace bb::world_state { +namespace azteclabs::wsdb::world_state { -using namespace bb::crypto::merkle_tree; +using namespace azteclabs::wsdb::merkle_tree; WorldState::WorldState(uint64_t thread_pool_size, const std::string& data_dir, @@ -177,7 +177,7 @@ void WorldState::create_canonical_fork(const std::string& dataDir, } { uint32_t levels = _tree_heights.at(MerkleTreeId::ARCHIVE); - std::vector initial_values{ compute_initial_block_header_hash( + std::vector initial_values{ compute_initial_block_header_hash( get_state_reference(WorldStateRevision::committed(), fork, true), _initial_header_generator_point, _genesis_timestamp) }; @@ -527,7 +527,7 @@ void WorldState::update_public_data(const PublicDataLeafValue& new_value, Fork:: } void WorldState::update_archive(const StateReference& block_state_ref, - const bb::fr& block_header_hash, + const azteclabs::wsdb::fr& block_header_hash, Fork::Id fork_id) { if (is_same_state_reference(WorldStateRevision{ .forkId = fork_id, .includeUncommitted = true }, block_state_ref)) { @@ -598,13 +598,13 @@ void WorldState::rollback() } WorldStateStatusFull WorldState::sync_block(const StateReference& block_state_ref, - const bb::fr& block_header_hash, - const std::vector& notes, - const std::vector& l1_to_l2_messages, - const std::vector& nullifiers, - const std::vector& public_writes, - const std::optional& expected_archive_root, - const std::optional& expected_previous_archive_root) + const azteclabs::wsdb::fr& block_header_hash, + const std::vector& notes, + const std::vector& l1_to_l2_messages, + const std::vector& nullifiers, + const std::vector& public_writes, + const std::optional& expected_archive_root, + const std::optional& expected_previous_archive_root) { validate_trees_are_equally_synched(); rollback(); @@ -617,7 +617,7 @@ WorldStateStatusFull WorldState::sync_block(const StateReference& block_state_re // block's lastArchive) and after (the resulting root must equal the block's archive), failing before commit // so the divergence is never persisted. if (expected_previous_archive_root.has_value()) { - const bb::fr actual_previous_archive_root = + const azteclabs::wsdb::fr actual_previous_archive_root = get_tree_info(WorldStateRevision::committed(), MerkleTreeId::ARCHIVE).meta.root; if (actual_previous_archive_root != expected_previous_archive_root.value()) { throw std::runtime_error( @@ -706,7 +706,7 @@ WorldStateStatusFull WorldState::sync_block(const StateReference& block_state_re // The archive tree is not part of the block state reference (see is_same_state_reference), so verify the // resulting archive root against the canonical block's archive root explicitly. if (expected_archive_root.has_value()) { - const bb::fr actual_archive_root = + const azteclabs::wsdb::fr actual_archive_root = get_tree_info(WorldStateRevision::uncommitted(), MerkleTreeId::ARCHIVE).meta.root; if (actual_archive_root != expected_archive_root.value()) { throw std::runtime_error( @@ -735,7 +735,7 @@ WorldStateStatusFull WorldState::sync_block(const StateReference& block_state_re GetLowIndexedLeafResponse WorldState::find_low_leaf_index(const WorldStateRevision& revision, MerkleTreeId tree_id, - const bb::fr& leaf_key) const + const azteclabs::wsdb::fr& leaf_key) const { Fork::SharedPtr fork = retrieve_fork(revision.forkId); Signal signal; @@ -1005,9 +1005,9 @@ bool WorldState::remove_historical_block(const block_number_t& blockNumber, Worl return true; } -bb::fr WorldState::compute_initial_block_header_hash(const StateReference& initial_state_ref, - uint32_t generator_point, - uint64_t genesis_timestamp) +azteclabs::wsdb::fr WorldState::compute_initial_block_header_hash(const StateReference& initial_state_ref, + uint32_t generator_point, + uint64_t genesis_timestamp) { // NOTE: this hash operations needs to match the one in // noir-project/noir-protocol-circuits/crates/types/src/abis/block_header.nr @@ -1027,22 +1027,22 @@ bb::fr WorldState::compute_initial_block_header_hash(const StateReference& initi initial_state_ref.at(MerkleTreeId::PUBLIC_DATA_TREE).second, 0, // sponge_blob_hash // global variables - 0, // chain_id - 0, // version - 0, // block_number - 0, // slot_number - bb::fr(genesis_timestamp), // timestamp - 0, // coinbase - 0, // fee_recipient - 0, // gas_fee.fee_per_da_gas - 0, // gas_fee.fee_per_l2_gas + 0, // chain_id + 0, // version + 0, // block_number + 0, // slot_number + azteclabs::wsdb::fr(genesis_timestamp), // timestamp + 0, // coinbase + 0, // fee_recipient + 0, // gas_fee.fee_per_da_gas + 0, // gas_fee.fee_per_l2_gas // total fees 0, // total mana used 0 }); } -bool WorldState::is_archive_tip(const WorldStateRevision& revision, const bb::fr& block_header_hash) const +bool WorldState::is_archive_tip(const WorldStateRevision& revision, const azteclabs::wsdb::fr& block_header_hash) const { std::vector> indices; @@ -1322,4 +1322,4 @@ WorldStateStatusFull WorldState::attempt_tree_resync() return status; } -} // namespace bb::world_state +} // namespace azteclabs::wsdb::world_state diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp b/native-packages/wsdb/cpp/src/world_state/world_state.hpp similarity index 87% rename from barretenberg/cpp/src/barretenberg/world_state/world_state.hpp rename to native-packages/wsdb/cpp/src/world_state/world_state.hpp index a5c82b9a8313..3b5c20e0d28c 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp +++ b/native-packages/wsdb/cpp/src/world_state/world_state.hpp @@ -1,42 +1,44 @@ #pragma once -#include "barretenberg/common/thread_pool.hpp" -#include "barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" -#include "barretenberg/crypto/merkle_tree/hash.hpp" -#include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" -#include "barretenberg/crypto/merkle_tree/response.hpp" -#include "barretenberg/crypto/merkle_tree/signal.hpp" -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/serialize/msgpack.hpp" -#include "barretenberg/world_state/fork.hpp" -#include "barretenberg/world_state/tree_with_store.hpp" -#include "barretenberg/world_state/types.hpp" -#include "barretenberg/world_state/world_state_stores.hpp" +#include "common/thread_pool.hpp" +#include "field/field_element.hpp" +#include "field/hash_policy.hpp" +#include "field/merkle_hash_policy_for.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" +#include "merkle_tree/hash_path.hpp" +#include "merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp" +#include "merkle_tree/node_store/cached_content_addressed_tree_store.hpp" +#include "merkle_tree/response.hpp" +#include "merkle_tree/signal.hpp" +#include "merkle_tree/tree_db_stats.hpp" +#include "merkle_tree/tree_meta.hpp" +#include "merkle_tree/types.hpp" +#include "world_state/fork.hpp" +#include "world_state/tree_with_store.hpp" +#include "world_state/types.hpp" +#include "world_state/world_state_stores.hpp" #include #include #include #include #include +#include #include #include #include #include #include -namespace bb::world_state { +namespace azteclabs::wsdb::world_state { -using crypto::merkle_tree::index_t; +using merkle_tree::index_t; // BatchInsertionResult and SequentialInsertionResult are defined in crypto/merkle_tree/response.hpp // (alongside LeafUpdateWitnessData) so the AVM can use them without depending on world_state. -using crypto::merkle_tree::BatchInsertionResult; -using crypto::merkle_tree::SequentialInsertionResult; +using merkle_tree::BatchInsertionResult; +using merkle_tree::SequentialInsertionResult; const uint64_t DEFAULT_MIN_NUMBER_OF_READERS = 128; @@ -108,7 +110,7 @@ class WorldState { * @param tree_id The ID of the tree * @return TreeInfo */ - crypto::merkle_tree::TreeMetaResponse get_tree_info(const WorldStateRevision& revision, MerkleTreeId tree_id) const; + merkle_tree::TreeMetaResponse get_tree_info(const WorldStateRevision& revision, MerkleTreeId tree_id) const; /** * @brief Gets the state reference for all the trees in the world state @@ -131,11 +133,11 @@ class WorldState { * @param revision The revision to query * @param tree_id The ID of the tree * @param leaf_index The index of the leaf - * @return crypto::merkle_tree::fr_sibling_path + * @return merkle_tree::fr_sibling_path */ - crypto::merkle_tree::fr_sibling_path get_sibling_path(const WorldStateRevision& revision, - MerkleTreeId tree_id, - index_t leaf_index) const; + merkle_tree::fr_sibling_path get_sibling_path(const WorldStateRevision& revision, + MerkleTreeId tree_id, + index_t leaf_index) const; void get_block_numbers_for_leaf_indices(const WorldStateRevision& revision, MerkleTreeId tree_id, @@ -152,14 +154,14 @@ class WorldState { * @return std::optional The IndexedLeaf object or nullopt if the leaf does not exist */ template - std::optional> get_indexed_leaf(const WorldStateRevision& revision, - MerkleTreeId tree_id, - index_t leaf_index) const; + std::optional> get_indexed_leaf(const WorldStateRevision& revision, + MerkleTreeId tree_id, + index_t leaf_index) const; /** * @brief Gets the value of a leaf in a tree * - * @tparam T the type of the leaf. Either bb::fr, NullifierLeafValue, PublicDataLeafValue + * @tparam T the type of the leaf. Either azteclabs::wsdb::fr, NullifierLeafValue, PublicDataLeafValue * @param revision The revision to query * @param tree_id The ID of the tree * @param leaf_index The index of the leaf @@ -177,9 +179,9 @@ class WorldState { * @param leaf_key The leaf to find the predecessor of * @return GetLowIndexedLeafResponse */ - crypto::merkle_tree::GetLowIndexedLeafResponse find_low_leaf_index(const WorldStateRevision& revision, - MerkleTreeId tree_id, - const bb::fr& leaf_key) const; + merkle_tree::GetLowIndexedLeafResponse find_low_leaf_index(const WorldStateRevision& revision, + MerkleTreeId tree_id, + const azteclabs::wsdb::fr& leaf_key) const; /** * @brief Finds the index of a leaf in a tree @@ -253,8 +255,7 @@ class WorldState { * * @param new_value The new value of the leaf. */ - void update_public_data(const crypto::merkle_tree::PublicDataLeafValue& new_value, - Fork::Id fork_id = CANONICAL_FORK_ID); + void update_public_data(const merkle_tree::PublicDataLeafValue& new_value, Fork::Id fork_id = CANONICAL_FORK_ID); /** * @brief Updates the archive tree with a new block. @@ -264,7 +265,7 @@ class WorldState { * @param fork_id The fork ID to update. */ void update_archive(const StateReference& block_state_ref, - const bb::fr& block_header_hash, + const azteclabs::wsdb::fr& block_header_hash, Fork::Id fork_id = CANONICAL_FORK_ID); /** @@ -285,14 +286,15 @@ class WorldState { WorldStateStatusFull remove_historical_blocks(const block_number_t& toBlockNumber); void get_status_summary(WorldStateStatusSummary& status) const; - WorldStateStatusFull sync_block(const StateReference& block_state_ref, - const bb::fr& block_header_hash, - const std::vector& notes, - const std::vector& l1_to_l2_messages, - const std::vector& nullifiers, - const std::vector& public_writes, - const std::optional& expected_archive_root = std::nullopt, - const std::optional& expected_previous_archive_root = std::nullopt); + WorldStateStatusFull sync_block( + const StateReference& block_state_ref, + const azteclabs::wsdb::fr& block_header_hash, + const std::vector& notes, + const std::vector& l1_to_l2_messages, + const std::vector& nullifiers, + const std::vector& public_writes, + const std::optional& expected_archive_root = std::nullopt, + const std::optional& expected_previous_archive_root = std::nullopt); uint32_t checkpoint(const uint64_t& forkId); void commit_checkpoint(const uint64_t& forkId); @@ -301,7 +303,7 @@ class WorldState { void revert_all_checkpoints_to(const uint64_t& forkId, uint32_t depth); private: - std::shared_ptr _workers; + std::shared_ptr _workers; WorldStateStores::Ptr _persistentStores; std::unordered_map _tree_heights; @@ -336,12 +338,12 @@ class WorldState { static bool block_state_matches_world_state(const StateReference& block_state_ref, const StateReference& tree_state_ref); - bool is_archive_tip(const WorldStateRevision& revision, const bb::fr& block_header_hash) const; + bool is_archive_tip(const WorldStateRevision& revision, const azteclabs::wsdb::fr& block_header_hash) const; bool is_same_state_reference(const WorldStateRevision& revision, const StateReference& state_ref) const; - static bb::fr compute_initial_block_header_hash(const StateReference& initial_state_ref, - uint32_t generator_point, - uint64_t genesis_timestamp = 0); + static azteclabs::wsdb::fr compute_initial_block_header_hash(const StateReference& initial_state_ref, + uint32_t generator_point, + uint64_t genesis_timestamp = 0); static StateReference get_state_reference(const WorldStateRevision& revision, Fork::SharedPtr fork, @@ -441,9 +443,9 @@ void WorldState::remove_historic_block_for_tree(TreeDBStats& dbStats, } template -std::optional> WorldState::get_indexed_leaf(const WorldStateRevision& rev, - MerkleTreeId id, - index_t leaf) const +std::optional> WorldState::get_indexed_leaf(const WorldStateRevision& rev, + MerkleTreeId id, + index_t leaf) const { using Store = ContentAddressedCachedTreeStore; using Tree = ContentAddressedIndexedTree>; @@ -481,7 +483,7 @@ std::optional WorldState::get_leaf(const WorldStateRevision& revision, MerkleTreeId tree_id, index_t leaf_index) const { - using namespace crypto::merkle_tree; + using namespace merkle_tree; Fork::SharedPtr fork = retrieve_fork(revision.forkId); @@ -489,7 +491,7 @@ std::optional WorldState::get_leaf(const WorldStateRevision& revision, bool success = true; std::string error_msg; Signal signal; - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { const auto& wrapper = std::get>(fork->_trees.at(tree_id)); auto callback = [&signal, &leaf, &success, &error_msg](const TypedResponse& response) { if (!response.success || !response.inner.leaf.has_value()) { @@ -543,7 +545,7 @@ void WorldState::find_leaf_indices(const WorldStateRevision& rev, std::vector>& indices, index_t start_index) const { - using namespace crypto::merkle_tree; + using namespace merkle_tree; Fork::SharedPtr fork = retrieve_fork(rev.forkId); TypedResponse local; @@ -553,7 +555,7 @@ void WorldState::find_leaf_indices(const WorldStateRevision& rev, local = std::move(response); signal.signal_level(0); }; - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { const auto& wrapper = std::get>(fork->_trees.at(id)); if (rev.is_historical()) { wrapper.tree->find_leaf_indices_from( @@ -590,7 +592,7 @@ void WorldState::find_sibling_paths(const WorldStateRevision& rev, const std::vector& leaves, std::vector>& paths) const { - using namespace crypto::merkle_tree; + using namespace merkle_tree; Fork::SharedPtr fork = retrieve_fork(rev.forkId); TypedResponse local; @@ -600,7 +602,7 @@ void WorldState::find_sibling_paths(const WorldStateRevision& rev, local = std::move(response); signal.signal_level(0); }; - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { const auto& wrapper = std::get>(fork->_trees.at(id)); if (rev.is_historical()) { wrapper.tree->find_leaf_sibling_paths(leaves, rev.blockNumber, rev.includeUncommitted, callback); @@ -631,7 +633,7 @@ void WorldState::find_sibling_paths(const WorldStateRevision& rev, template void WorldState::append_leaves(MerkleTreeId id, const std::vector& leaves, Fork::Id fork_id) { - using namespace crypto::merkle_tree; + using namespace merkle_tree; Fork::SharedPtr fork = retrieve_fork(fork_id); @@ -640,7 +642,7 @@ template void WorldState::append_leaves(MerkleTreeId id, const std: bool success = true; std::string error_msg; - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { auto& wrapper = std::get>(fork->_trees.at(id)); auto callback = [&](const auto& resp) { if (!resp.success) { @@ -677,7 +679,7 @@ BatchInsertionResult WorldState::batch_insert_indexed_leaves(MerkleTreeId id, uint32_t subtree_depth, Fork::Id fork_id) { - using namespace crypto::merkle_tree; + using namespace merkle_tree; using Store = ContentAddressedCachedTreeStore; using Tree = ContentAddressedIndexedTree>; @@ -717,7 +719,7 @@ SequentialInsertionResult WorldState::insert_indexed_leaves(MerkleTreeId id, const std::vector& leaves, Fork::Id fork_id) { - using namespace crypto::merkle_tree; + using namespace merkle_tree; using Store = ContentAddressedCachedTreeStore; using Tree = ContentAddressedIndexedTree>; @@ -750,6 +752,6 @@ SequentialInsertionResult WorldState::insert_indexed_leaves(MerkleTreeId id, return result; } -} // namespace bb::world_state +} // namespace azteclabs::wsdb::world_state // MSGPACK_ADD_ENUM(MerkleTreeId) lives in crypto/merkle_tree/merkle_tree_id.hpp. diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp b/native-packages/wsdb/cpp/src/world_state/world_state.test.cpp similarity index 91% rename from barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp rename to native-packages/wsdb/cpp/src/world_state/world_state.test.cpp index 4b6181fc9c98..05419416b4cf 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp +++ b/native-packages/wsdb/cpp/src/world_state/world_state.test.cpp @@ -1,12 +1,12 @@ -#include "barretenberg/world_state/world_state.hpp" -#include "barretenberg/aztec/aztec_constants.hpp" -#include "barretenberg/crypto/merkle_tree/fixtures.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" -#include "barretenberg/crypto/merkle_tree/response.hpp" -#include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/world_state/fork.hpp" -#include "barretenberg/world_state/types.hpp" +#include "world_state/world_state.hpp" +#include "common/aztec_constants.hpp" +#include "field/field_element.hpp" +#include "merkle_tree/fixtures.hpp" +#include "merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/response.hpp" +#include "merkle_tree/tree_meta.hpp" +#include "world_state/fork.hpp" +#include "world_state/types.hpp" #include #include #include @@ -16,8 +16,8 @@ #include #include -using namespace bb::world_state; -using namespace bb::crypto::merkle_tree; +using namespace azteclabs::wsdb::world_state; +using namespace azteclabs::wsdb::merkle_tree; class WorldStateTest : public testing::Test { protected: @@ -174,28 +174,32 @@ TEST_F(WorldStateTest, GetInitialTreeInfoForAllTrees) auto info = ws.get_tree_info(WorldStateRevision::committed(), MerkleTreeId::NULLIFIER_TREE); EXPECT_EQ(info.meta.size, 128); EXPECT_EQ(info.meta.depth, tree_heights.at(MerkleTreeId::NULLIFIER_TREE)); - EXPECT_EQ(info.meta.root, bb::fr("0x18935581a8ed73d08ffd00386fba55ba6c89f3ab848a76b8fedfa9034cee0454")); + EXPECT_EQ(info.meta.root, + azteclabs::wsdb::fr("0x18935581a8ed73d08ffd00386fba55ba6c89f3ab848a76b8fedfa9034cee0454")); } { auto info = ws.get_tree_info(WorldStateRevision::committed(), MerkleTreeId::NOTE_HASH_TREE); EXPECT_EQ(info.meta.size, 0); EXPECT_EQ(info.meta.depth, tree_heights.at(MerkleTreeId::NOTE_HASH_TREE)); - EXPECT_EQ(info.meta.root, bb::fr("0x2590f2aab19dd791700b4a43d3f52bb88ef2409a3731da8e848663559202e4c6")); + EXPECT_EQ(info.meta.root, + azteclabs::wsdb::fr("0x2590f2aab19dd791700b4a43d3f52bb88ef2409a3731da8e848663559202e4c6")); } { auto info = ws.get_tree_info(WorldStateRevision::committed(), MerkleTreeId::PUBLIC_DATA_TREE); EXPECT_EQ(info.meta.size, 128); EXPECT_EQ(info.meta.depth, tree_heights.at(MerkleTreeId::PUBLIC_DATA_TREE)); - EXPECT_EQ(info.meta.root, bb::fr("0x1bef38b621017d3c7416663d0cd81369424560710526a3fbaaec13e356b9d084")); + EXPECT_EQ(info.meta.root, + azteclabs::wsdb::fr("0x1bef38b621017d3c7416663d0cd81369424560710526a3fbaaec13e356b9d084")); } { auto info = ws.get_tree_info(WorldStateRevision::committed(), MerkleTreeId::L1_TO_L2_MESSAGE_TREE); EXPECT_EQ(info.meta.size, 0); EXPECT_EQ(info.meta.depth, tree_heights.at(MerkleTreeId::L1_TO_L2_MESSAGE_TREE)); - EXPECT_EQ(info.meta.root, bb::fr("0x0fef6d80d31109ddb56d6b3f607cbc9c0af0bff3ea0d43e8f278983c64c11f7a")); + EXPECT_EQ(info.meta.root, + azteclabs::wsdb::fr("0x0fef6d80d31109ddb56d6b3f607cbc9c0af0bff3ea0d43e8f278983c64c11f7a")); } { @@ -203,11 +207,14 @@ TEST_F(WorldStateTest, GetInitialTreeInfoForAllTrees) EXPECT_EQ(info.meta.size, 1); EXPECT_EQ(info.meta.depth, tree_heights.at(MerkleTreeId::ARCHIVE)); // this is the expected archive tree root at genesis - EXPECT_EQ(info.meta.root, bb::fr(GENESIS_ARCHIVE_ROOT)); + EXPECT_EQ(info.meta.root, azteclabs::wsdb::fr(GENESIS_ARCHIVE_ROOT)); // The leaf at index 0 is the genesis block hash. - assert_leaf_value( - ws, WorldStateRevision::committed(), MerkleTreeId::ARCHIVE, 0, bb::fr(GENESIS_BLOCK_HEADER_HASH)); + assert_leaf_value(ws, + WorldStateRevision::committed(), + MerkleTreeId::ARCHIVE, + 0, + azteclabs::wsdb::fr(GENESIS_BLOCK_HEADER_HASH)); } } @@ -286,61 +293,69 @@ TEST_F(WorldStateTest, GetStateReference) auto snapshot = state_ref.at(MerkleTreeId::NULLIFIER_TREE); EXPECT_EQ( snapshot, - std::make_pair(bb::fr("0x18935581a8ed73d08ffd00386fba55ba6c89f3ab848a76b8fedfa9034cee0454"), 128UL)); + std::make_pair( + azteclabs::wsdb::fr("0x18935581a8ed73d08ffd00386fba55ba6c89f3ab848a76b8fedfa9034cee0454"), 128UL)); } { auto snapshot = state_ref.at(MerkleTreeId::NOTE_HASH_TREE); EXPECT_EQ( snapshot, - std::make_pair(bb::fr("0x2590f2aab19dd791700b4a43d3f52bb88ef2409a3731da8e848663559202e4c6"), 0UL)); + std::make_pair( + azteclabs::wsdb::fr("0x2590f2aab19dd791700b4a43d3f52bb88ef2409a3731da8e848663559202e4c6"), 0UL)); } { auto snapshot = state_ref.at(MerkleTreeId::PUBLIC_DATA_TREE); EXPECT_EQ( snapshot, - std::make_pair(bb::fr("0x1bef38b621017d3c7416663d0cd81369424560710526a3fbaaec13e356b9d084"), 128UL)); + std::make_pair( + azteclabs::wsdb::fr("0x1bef38b621017d3c7416663d0cd81369424560710526a3fbaaec13e356b9d084"), 128UL)); } { auto snapshot = state_ref.at(MerkleTreeId::L1_TO_L2_MESSAGE_TREE); EXPECT_EQ( snapshot, - std::make_pair(bb::fr("0x0fef6d80d31109ddb56d6b3f607cbc9c0af0bff3ea0d43e8f278983c64c11f7a"), 0UL)); + std::make_pair( + azteclabs::wsdb::fr("0x0fef6d80d31109ddb56d6b3f607cbc9c0af0bff3ea0d43e8f278983c64c11f7a"), 0UL)); } } { - ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { 1 }); + ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { 1 }); auto state_ref = ws.get_state_reference(WorldStateRevision::uncommitted()); { auto snapshot = state_ref.at(MerkleTreeId::NULLIFIER_TREE); EXPECT_EQ( snapshot, - std::make_pair(bb::fr("0x18935581a8ed73d08ffd00386fba55ba6c89f3ab848a76b8fedfa9034cee0454"), 128UL)); + std::make_pair( + azteclabs::wsdb::fr("0x18935581a8ed73d08ffd00386fba55ba6c89f3ab848a76b8fedfa9034cee0454"), 128UL)); } { auto snapshot = state_ref.at(MerkleTreeId::NOTE_HASH_TREE); EXPECT_EQ( snapshot, - std::make_pair(bb::fr("0x278449ce779093eaaa1bd9fc608f7ed7b595417710e2baf359e1488650967f47"), 1UL)); + std::make_pair( + azteclabs::wsdb::fr("0x278449ce779093eaaa1bd9fc608f7ed7b595417710e2baf359e1488650967f47"), 1UL)); } { auto snapshot = state_ref.at(MerkleTreeId::PUBLIC_DATA_TREE); EXPECT_EQ( snapshot, - std::make_pair(bb::fr("0x1bef38b621017d3c7416663d0cd81369424560710526a3fbaaec13e356b9d084"), 128UL)); + std::make_pair( + azteclabs::wsdb::fr("0x1bef38b621017d3c7416663d0cd81369424560710526a3fbaaec13e356b9d084"), 128UL)); } { auto snapshot = state_ref.at(MerkleTreeId::L1_TO_L2_MESSAGE_TREE); EXPECT_EQ( snapshot, - std::make_pair(bb::fr("0x0fef6d80d31109ddb56d6b3f607cbc9c0af0bff3ea0d43e8f278983c64c11f7a"), 0UL)); + std::make_pair( + azteclabs::wsdb::fr("0x0fef6d80d31109ddb56d6b3f607cbc9c0af0bff3ea0d43e8f278983c64c11f7a"), 0UL)); } } } @@ -350,7 +365,7 @@ TEST_F(WorldStateTest, GetInitialStateReference) WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); auto before_commit = ws.get_initial_state_reference(); - ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { 1 }); + ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { 1 }); WorldStateStatusFull status; ws.commit(status); @@ -654,8 +669,8 @@ TEST_F(WorldStateTest, SyncBlockRejectsDivergentArchiveRoot) }; // Learn the canonical previous (genesis) and resulting archive roots from an untracked sync. - bb::fr previous_root; - bb::fr resulting_root; + azteclabs::wsdb::fr previous_root; + azteclabs::wsdb::fr resulting_root; { WorldState scratch( thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); @@ -677,7 +692,9 @@ TEST_F(WorldStateTest, SyncBlockRejectsDivergentArchiveRoot) { NullifierLeafValue(144) }, { { PublicDataLeafValue(145, 1) } }, resulting_root, - previous_root + fr(1)), + // A root that is not the committed previous root; the two learned roots differ, + // so `resulting_root` is a valid "wrong previous root" without needing fr arithmetic. + resulting_root), std::runtime_error); // A wrong resulting archive root is rejected before commit. @@ -687,7 +704,9 @@ TEST_F(WorldStateTest, SyncBlockRejectsDivergentArchiveRoot) { 43 }, { NullifierLeafValue(144) }, { { PublicDataLeafValue(145, 1) } }, - resulting_root + fr(1), + // A root that is not the resulting archive root; `previous_root` differs from + // `resulting_root`, so it is a valid "wrong resulting root" without fr arithmetic. + previous_root, previous_root), std::runtime_error); @@ -753,7 +772,7 @@ TEST_F(WorldStateTest, SyncBlockFromDirtyState) TEST_F(WorldStateTest, SyncCurrentBlock) { WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); - bb::fr block_hash(1); + azteclabs::wsdb::fr block_hash(1); StateReference block_state_ref = { { MerkleTreeId::NULLIFIER_TREE, { fr("0x2e2e2d8b72294a440c728a646f01476624063f0b50dcfe293cc0fc26bef9e311"), 129 } }, @@ -880,7 +899,7 @@ TEST_F(WorldStateTest, ForkingAtBlock0AndAdvancingFork) }, MerkleTreeId::ARCHIVE); - ws.append_leaves(MerkleTreeId::ARCHIVE, { fr(1) }, fork_id); + ws.append_leaves(MerkleTreeId::ARCHIVE, { fr(1) }, fork_id); auto canonical_archive_state_after = ws.get_tree_info(WorldStateRevision::uncommitted(), MerkleTreeId::ARCHIVE); auto fork_archive_state_after = ws.get_tree_info( @@ -909,8 +928,8 @@ TEST_F(WorldStateTest, ForkingAtBlock0AndAdvancingCanonicalState) MerkleTreeId::ARCHIVE); // fork the state - ws.append_leaves(MerkleTreeId::ARCHIVE, { fr(1) }); - ws.append_leaves(MerkleTreeId::ARCHIVE, { fr(2) }, fork_id); + ws.append_leaves(MerkleTreeId::ARCHIVE, { fr(1) }); + ws.append_leaves(MerkleTreeId::ARCHIVE, { fr(2) }, fork_id); auto canonical_archive_state_after_insert = ws.get_tree_info(WorldStateRevision::uncommitted(), MerkleTreeId::ARCHIVE); @@ -947,9 +966,10 @@ TEST_F(WorldStateTest, ForkingAtBlock0AndAdvancingCanonicalState) EXPECT_EQ(canonical_archive_state_after_commit.meta.size, canonical_archive_state_after_insert.meta.size); // canonical should have value 1 as the first leaf (committed state) - assert_leaf_value(ws, WorldStateRevision{ .includeUncommitted = false }, MerkleTreeId::ARCHIVE, 1, 1); + assert_leaf_value( + ws, WorldStateRevision{ .includeUncommitted = false }, MerkleTreeId::ARCHIVE, 1, 1); // fork should still have value 2 as the first leaf (uncommitted) - assert_leaf_value( + assert_leaf_value( ws, WorldStateRevision{ .forkId = fork_id, .includeUncommitted = true }, MerkleTreeId::ARCHIVE, 1, 2); } @@ -958,8 +978,8 @@ TEST_F(WorldStateTest, BuildsABlockInAFork) WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); auto fork_id = ws.create_fork(0); - ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { 42 }, fork_id); - ws.append_leaves(MerkleTreeId::L1_TO_L2_MESSAGE_TREE, { 43 }, fork_id); + ws.append_leaves(MerkleTreeId::NOTE_HASH_TREE, { 42 }, fork_id); + ws.append_leaves(MerkleTreeId::L1_TO_L2_MESSAGE_TREE, { 43 }, fork_id); ws.batch_insert_indexed_leaves(MerkleTreeId::NULLIFIER_TREE, { { 129 } }, 0, fork_id); ws.batch_insert_indexed_leaves(MerkleTreeId::PUBLIC_DATA_TREE, { { 129, 1 } }, 0, fork_id); @@ -979,7 +999,7 @@ TEST_F(WorldStateTest, BuildsABlockInAFork) TEST_F(WorldStateTest, GetBlockForIndex) { WorldState ws(thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); - // bb::fr block_hash(1); + // azteclabs::wsdb::fr block_hash(1); StateReference block_state_ref = { { MerkleTreeId::NULLIFIER_TREE, { fr("0x2e2e2d8b72294a440c728a646f01476624063f0b50dcfe293cc0fc26bef9e311"), 129 } }, @@ -1022,8 +1042,8 @@ TEST_F(WorldStateTest, GetBlockForIndex) TEST_F(WorldStateTest, SyncEmptyBlockAcceptsBogusHashWithoutArchiveCheck) { // Learn the canonical previous and resulting archive roots from an empty-block sync on a scratch instance. - bb::fr previous_archive_root; - bb::fr canonical_archive_root; + azteclabs::wsdb::fr previous_archive_root; + azteclabs::wsdb::fr canonical_archive_root; { WorldState scratch( thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); @@ -1043,7 +1063,8 @@ TEST_F(WorldStateTest, SyncEmptyBlockAcceptsBogusHashWithoutArchiveCheck) // Sync an empty block using a bogus hash — no expected-archive-root arguments provided. EXPECT_NO_THROW(ws.sync_block(genesis_state_ref, fr(42), {}, {}, {}, {})); - bb::fr actual_archive_root = ws.get_tree_info(WorldStateRevision::committed(), MerkleTreeId::ARCHIVE).meta.root; + azteclabs::wsdb::fr actual_archive_root = + ws.get_tree_info(WorldStateRevision::committed(), MerkleTreeId::ARCHIVE).meta.root; // The bogus hash committed successfully, so the resulting archive root differs from the canonical one. EXPECT_NE(actual_archive_root, canonical_archive_root); @@ -1055,8 +1076,8 @@ TEST_F(WorldStateTest, SyncEmptyBlockAcceptsBogusHashWithoutArchiveCheck) TEST_F(WorldStateTest, SyncEmptyBlockRejectsBogusHashWhenArchiveRootsAreChecked) { // Learn the canonical previous and resulting archive roots from an empty-block sync on a scratch instance. - bb::fr previous_archive_root; - bb::fr canonical_archive_root; + azteclabs::wsdb::fr previous_archive_root; + azteclabs::wsdb::fr canonical_archive_root; { WorldState scratch( thread_pool_size, data_dir, map_size, tree_heights, tree_prefill, initial_header_generator_point); diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state_stores.hpp b/native-packages/wsdb/cpp/src/world_state/world_state_stores.hpp similarity index 89% rename from barretenberg/cpp/src/barretenberg/world_state/world_state_stores.hpp rename to native-packages/wsdb/cpp/src/world_state/world_state_stores.hpp index 31c924d0a0cb..14327cd2df52 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state_stores.hpp +++ b/native-packages/wsdb/cpp/src/world_state/world_state_stores.hpp @@ -1,12 +1,12 @@ #pragma once -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" #include #include -using namespace bb::crypto::merkle_tree; +using namespace azteclabs::wsdb::merkle_tree; -namespace bb::world_state { +namespace azteclabs::wsdb::world_state { struct WorldStateStores { using Ptr = std::shared_ptr; @@ -50,4 +50,4 @@ struct WorldStateStores { nullifierStore, publicDataStore, archiveStore, noteHashStore, messageStore }; }; -} // namespace bb::world_state \ No newline at end of file +} // namespace azteclabs::wsdb::world_state \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/wsdb/cli.cpp b/native-packages/wsdb/cpp/src/wsdb/cli.cpp similarity index 88% rename from barretenberg/cpp/src/barretenberg/wsdb/cli.cpp rename to native-packages/wsdb/cpp/src/wsdb/cli.cpp index 2cf90fb38bcb..7ac16a136cfb 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb/cli.cpp +++ b/native-packages/wsdb/cpp/src/wsdb/cli.cpp @@ -1,22 +1,22 @@ -#include "barretenberg/wsdb/cli.hpp" -#include "barretenberg/common/log.hpp" -#include "barretenberg/common/throw_or_abort.hpp" -#include "barretenberg/serialize/msgpack.hpp" -#include "barretenberg/world_state/world_state.hpp" -#include "barretenberg/wsdb/generated/wsdb_ipc_server.hpp" -#include "barretenberg/wsdb/wsdb_ipc_server.hpp" - -#include "barretenberg/bb/deps/cli11.hpp" +#include "wsdb/cli.hpp" +#include "common/assert.hpp" +#include "common/log.hpp" +#include +#include "world_state/world_state.hpp" +#include "wsdb/generated/wsdb_ipc_server.hpp" +#include "wsdb/wsdb_ipc_server.hpp" + +#include #include #include #include #include #include -namespace bb::wsdb { +namespace azteclabs::wsdb { -using namespace bb::world_state; -using namespace bb::crypto::merkle_tree; +using namespace azteclabs::wsdb::world_state; +using namespace azteclabs::wsdb::merkle_tree; int parse_and_run_wsdb(int argc, char* argv[]) { @@ -110,4 +110,4 @@ int parse_and_run_wsdb(int argc, char* argv[]) return 0; } -} // namespace bb::wsdb +} // namespace azteclabs::wsdb diff --git a/barretenberg/cpp/src/barretenberg/wsdb/cli.hpp b/native-packages/wsdb/cpp/src/wsdb/cli.hpp similarity index 52% rename from barretenberg/cpp/src/barretenberg/wsdb/cli.hpp rename to native-packages/wsdb/cpp/src/wsdb/cli.hpp index 02b8705a5f4d..68e4632db8bb 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb/cli.hpp +++ b/native-packages/wsdb/cpp/src/wsdb/cli.hpp @@ -1,7 +1,7 @@ #pragma once -namespace bb::wsdb { +namespace azteclabs::wsdb { int parse_and_run_wsdb(int argc, char* argv[]); -} // namespace bb::wsdb +} // namespace azteclabs::wsdb diff --git a/native-packages/wsdb/cpp/src/wsdb/main.cpp b/native-packages/wsdb/cpp/src/wsdb/main.cpp new file mode 100644 index 000000000000..7e92494bfde7 --- /dev/null +++ b/native-packages/wsdb/cpp/src/wsdb/main.cpp @@ -0,0 +1,6 @@ +#include "wsdb/cli.hpp" + +int main(int argc, char* argv[]) +{ + return azteclabs::wsdb::parse_and_run_wsdb(argc, argv); +} diff --git a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/memory_merkle_db.test.cpp b/native-packages/wsdb/cpp/src/wsdb/memory_merkle_db.test.cpp similarity index 98% rename from barretenberg/cpp/src/barretenberg/vm2/simulation/lib/memory_merkle_db.test.cpp rename to native-packages/wsdb/cpp/src/wsdb/memory_merkle_db.test.cpp index 94c34a0cb28a..6e43cbf66635 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/memory_merkle_db.test.cpp +++ b/native-packages/wsdb/cpp/src/wsdb/memory_merkle_db.test.cpp @@ -8,11 +8,11 @@ #include #include "barretenberg/aztec/aztec_constants.hpp" -#include "barretenberg/crypto/merkle_tree/fixtures.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "merkle_tree/fixtures.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/merkle_tree_id.hpp" #include "barretenberg/vm2/common/field.hpp" -#include "barretenberg/world_state/world_state.hpp" +#include "world_state/world_state.hpp" // This test is the canonical-fidelity gate for MemoryMerkleDB. It drives an ephemeral, file-backed // world_state::WorldState and an in-memory MemoryMerkleDB through an identical sequence of operations diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_handlers.cpp b/native-packages/wsdb/cpp/src/wsdb/wsdb_handlers.cpp similarity index 97% rename from barretenberg/cpp/src/barretenberg/wsdb/wsdb_handlers.cpp rename to native-packages/wsdb/cpp/src/wsdb/wsdb_handlers.cpp index e30cb2e213d7..fe40385053e8 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_handlers.cpp +++ b/native-packages/wsdb/cpp/src/wsdb/wsdb_handlers.cpp @@ -15,20 +15,21 @@ * into respond.ok(...) or a thrown exception into respond.error(...). Wire <-> * domain conversion happens inside the lambda via wsdb_wire_convert.hpp. */ -#include "barretenberg/wsdb/wsdb_handlers.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/response.hpp" -#include "barretenberg/world_state/world_state.hpp" -#include "barretenberg/wsdb/wsdb_schedule.hpp" -#include "barretenberg/wsdb/wsdb_wire_convert.hpp" +#include "wsdb/wsdb_handlers.hpp" +#include "merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/response.hpp" +#include "merkle_tree/tree_db_stats.hpp" +#include "world_state/world_state.hpp" +#include "wsdb/wsdb_schedule.hpp" +#include "wsdb/wsdb_wire_convert.hpp" #include #include -namespace bb::wsdb { +namespace azteclabs::wsdb { -using namespace bb::world_state; -using namespace bb::crypto::merkle_tree; +using namespace azteclabs::wsdb::world_state; +using namespace azteclabs::wsdb::merkle_tree; // --------------------------------------------------------------------------- // Tree info / state queries @@ -101,10 +102,10 @@ void handle_get_leaf_value(WsdbRequest& ctx, case world_state::MerkleTreeId::NOTE_HASH_TREE: case world_state::MerkleTreeId::L1_TO_L2_MESSAGE_TREE: case world_state::MerkleTreeId::ARCHIVE: { - auto leaf = ctx.world_state.get_leaf(revision, tree_id, leaf_index); - return wire::WsdbGetLeafValueResponse{ .value = leaf.has_value() - ? std::optional(fr_to_wire(*leaf)) - : std::nullopt }; + auto leaf = ctx.world_state.get_leaf(revision, tree_id, leaf_index); + return wire::WsdbGetLeafValueResponse{ + .value = leaf.has_value() ? std::optional(fr_to_wire(*leaf)) : std::nullopt + }; } default: throw std::runtime_error("Unsupported tree type for get_leaf_value"); @@ -260,7 +261,7 @@ void handle_find_leaf_indices(WsdbRequest& ctx, case world_state::MerkleTreeId::L1_TO_L2_MESSAGE_TREE: case world_state::MerkleTreeId::ARCHIVE: { auto typed_leaves = fr_vec_from_wire(cmd.leaves); - ctx.world_state.find_leaf_indices( + ctx.world_state.find_leaf_indices( revision, tree_id, typed_leaves, indices, start_index); break; } @@ -363,7 +364,8 @@ void handle_find_sibling_paths(WsdbRequest& ctx, case world_state::MerkleTreeId::L1_TO_L2_MESSAGE_TREE: case world_state::MerkleTreeId::ARCHIVE: { auto typed_leaves = fr_vec_from_wire(cmd.leaves); - ctx.world_state.find_sibling_paths(revision, tree_id, typed_leaves, paths); + ctx.world_state.find_sibling_paths( + revision, tree_id, typed_leaves, paths); break; } default: @@ -454,7 +456,7 @@ void handle_append_leaves(WsdbRequest& ctx, case world_state::MerkleTreeId::NOTE_HASH_TREE: case world_state::MerkleTreeId::L1_TO_L2_MESSAGE_TREE: case world_state::MerkleTreeId::ARCHIVE: { - ctx.world_state.append_leaves(tree_id, fr_vec_from_wire(cmd.leaves), cmd.forkId); + ctx.world_state.append_leaves(tree_id, fr_vec_from_wire(cmd.leaves), cmd.forkId); break; } default: @@ -618,11 +620,11 @@ void handle_sync_block(WsdbRequest& ctx, wire::WsdbSyncBlock&& cmd, Responder expected_archive_root; + std::optional expected_archive_root; if (cmd.expectedArchiveRoot.has_value()) { expected_archive_root = fr_from_wire(cmd.expectedArchiveRoot.value()); } - std::optional expected_previous_archive_root; + std::optional expected_previous_archive_root; if (cmd.expectedPreviousArchiveRoot.has_value()) { expected_previous_archive_root = fr_from_wire(cmd.expectedPreviousArchiveRoot.value()); } @@ -794,4 +796,4 @@ void handle_copy_stores(WsdbRequest& ctx, wire::WsdbCopyStores&& cmd, Responder< }); } -} // namespace bb::wsdb +} // namespace azteclabs::wsdb diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_handlers.hpp b/native-packages/wsdb/cpp/src/wsdb/wsdb_handlers.hpp similarity index 97% rename from barretenberg/cpp/src/barretenberg/wsdb/wsdb_handlers.hpp rename to native-packages/wsdb/cpp/src/wsdb/wsdb_handlers.hpp index 886cd0297611..d3e4eb710ae3 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_handlers.hpp +++ b/native-packages/wsdb/cpp/src/wsdb/wsdb_handlers.hpp @@ -14,11 +14,12 @@ * wsdb_schedule.hpp) and the response is produced by respond.ok()/error() when * the work completes. Definitions live in wsdb_handlers.cpp. */ -#include "barretenberg/wsdb/generated/wsdb_dispatch.hpp" // Responder -#include "barretenberg/wsdb/generated/wsdb_types.hpp" -#include "barretenberg/wsdb/wsdb_request.hpp" +#include "merkle_tree/tree_db_stats.hpp" +#include "wsdb/generated/wsdb_dispatch.hpp" // Responder +#include "wsdb/generated/wsdb_types.hpp" +#include "wsdb/wsdb_request.hpp" -namespace bb::wsdb { +namespace azteclabs::wsdb { void handle_get_tree_info(WsdbRequest& ctx, wire::WsdbGetTreeInfo&& cmd, @@ -127,4 +128,4 @@ void handle_revert_all_checkpoints(WsdbRequest& ctx, Responder respond); void handle_copy_stores(WsdbRequest& ctx, wire::WsdbCopyStores&& cmd, Responder respond); -} // namespace bb::wsdb +} // namespace azteclabs::wsdb diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_server.cpp b/native-packages/wsdb/cpp/src/wsdb/wsdb_ipc_server.cpp similarity index 92% rename from barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_server.cpp rename to native-packages/wsdb/cpp/src/wsdb/wsdb_ipc_server.cpp index ca98eb60e12e..dbbe69e2c2ae 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_server.cpp +++ b/native-packages/wsdb/cpp/src/wsdb/wsdb_ipc_server.cpp @@ -1,13 +1,13 @@ -#include "barretenberg/wsdb/wsdb_ipc_server.hpp" -#include "barretenberg/common/log.hpp" -#include "barretenberg/common/thread_pool.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/serialize/msgpack.hpp" -#include "barretenberg/world_state/world_state.hpp" -#include "barretenberg/wsdb/generated/wsdb_ipc_server.hpp" -#include "barretenberg/wsdb/wsdb_handlers.hpp" -#include "barretenberg/wsdb/wsdb_request.hpp" -#include "barretenberg/wsdb/wsdb_scheduler.hpp" +#include "wsdb/wsdb_ipc_server.hpp" +#include "common/log.hpp" +#include "common/thread_pool.hpp" +#include "merkle_tree/indexed_leaf.hpp" +#include +#include "world_state/world_state.hpp" +#include "wsdb/generated/wsdb_ipc_server.hpp" +#include "wsdb/wsdb_handlers.hpp" +#include "wsdb/wsdb_request.hpp" +#include "wsdb/wsdb_scheduler.hpp" #include "ipc_runtime/ipc_server.hpp" #include "ipc_runtime/serve_helper.hpp" #include "ipc_runtime/signal_handlers.hpp" @@ -22,10 +22,10 @@ #include #include -namespace bb::wsdb { +namespace azteclabs::wsdb { -using namespace bb::world_state; -using namespace bb::crypto::merkle_tree; +using namespace azteclabs::wsdb::world_state; +using namespace azteclabs::wsdb::merkle_tree; // --------------------------------------------------------------------------- // Simple JSON-like parsing for config maps @@ -216,7 +216,7 @@ int execute_wsdb_server(const std::string& input_path, // serialized through the single reactor thread. It is deliberately DISTINCT // from WorldState's own intra-op pool (the `threads` arg above): mutating // handlers enqueue subtasks onto that pool and wait() on them, so dispatching - // those handlers onto the SAME pool could deadlock (bb::ThreadPool is + // those handlers onto the SAME pool could deadlock (azteclabs::wsdb::ThreadPool is // blocking and non-work-stealing). // // Sized from the caller-provided `threads` budget (the same value used for @@ -225,7 +225,7 @@ int execute_wsdb_server(const std::string& input_path, // 2-CPU CI container), which would spawn a huge pool per wsdb process and // exhaust the per-UID thread limit (pthread_create EAGAIN). uint32_t dispatch_threads = std::max(2, threads); - bb::ThreadPool dispatch_pool(dispatch_threads); + azteclabs::wsdb::ThreadPool dispatch_pool(dispatch_threads); // Server-side ordering: the database, not the client, guarantees per-fork // read/write consistency. Each handler hands its work to this scheduler via @@ -246,4 +246,4 @@ int execute_wsdb_server(const std::string& input_path, return 0; } -} // namespace bb::wsdb +} // namespace azteclabs::wsdb diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_server.hpp b/native-packages/wsdb/cpp/src/wsdb/wsdb_ipc_server.hpp similarity index 93% rename from barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_server.hpp rename to native-packages/wsdb/cpp/src/wsdb/wsdb_ipc_server.hpp index afb03cba3512..5ab9d7b9e0e0 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_server.hpp +++ b/native-packages/wsdb/cpp/src/wsdb/wsdb_ipc_server.hpp @@ -4,7 +4,7 @@ #include #include -namespace bb::wsdb { +namespace azteclabs::wsdb { /** * @brief Start the aztec-wsdb IPC server. @@ -24,4 +24,4 @@ int execute_wsdb_server(const std::string& input_path, size_t request_ring_size, size_t response_ring_size); -} // namespace bb::wsdb +} // namespace azteclabs::wsdb diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_request.hpp b/native-packages/wsdb/cpp/src/wsdb/wsdb_request.hpp similarity index 87% rename from barretenberg/cpp/src/barretenberg/wsdb/wsdb_request.hpp rename to native-packages/wsdb/cpp/src/wsdb/wsdb_request.hpp index 530ab629fae3..38f651ec2470 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_request.hpp +++ b/native-packages/wsdb/cpp/src/wsdb/wsdb_request.hpp @@ -8,9 +8,9 @@ * work to (so reads run concurrently and writes are serialized per fork — see * wsdb_schedule.hpp / wsdb_scheduler.hpp). */ -#include "barretenberg/world_state/world_state.hpp" +#include "world_state/world_state.hpp" -namespace bb::wsdb { +namespace azteclabs::wsdb { class WsdbScheduler; @@ -21,4 +21,4 @@ struct WsdbRequest { WsdbScheduler* scheduler = nullptr; }; -} // namespace bb::wsdb +} // namespace azteclabs::wsdb diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_schedule.hpp b/native-packages/wsdb/cpp/src/wsdb/wsdb_schedule.hpp similarity index 89% rename from barretenberg/cpp/src/barretenberg/wsdb/wsdb_schedule.hpp rename to native-packages/wsdb/cpp/src/wsdb/wsdb_schedule.hpp index aa6e0d9e63a0..71e6a1c9b355 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_schedule.hpp +++ b/native-packages/wsdb/cpp/src/wsdb/wsdb_schedule.hpp @@ -12,16 +12,16 @@ * bodies stay free of threading and error-frame boilerplate. */ -#include "barretenberg/wsdb/generated/wsdb_dispatch.hpp" // Responder -#include "barretenberg/wsdb/wsdb_request.hpp" -#include "barretenberg/wsdb/wsdb_scheduler.hpp" +#include "wsdb/generated/wsdb_dispatch.hpp" // Responder +#include "wsdb/wsdb_request.hpp" +#include "wsdb/wsdb_scheduler.hpp" #include #include #include #include -namespace bb::wsdb { +namespace azteclabs::wsdb { namespace detail { template std::function wrap(Responder respond, Fn&& logic) @@ -53,4 +53,4 @@ void schedule_write(WsdbRequest& ctx, uint64_t fork, Responder respond, Fn ctx.scheduler->submit_write(fork, detail::wrap(std::move(respond), std::forward(logic))); } -} // namespace bb::wsdb +} // namespace azteclabs::wsdb diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_scheduler.hpp b/native-packages/wsdb/cpp/src/wsdb/wsdb_scheduler.hpp similarity index 96% rename from barretenberg/cpp/src/barretenberg/wsdb/wsdb_scheduler.hpp rename to native-packages/wsdb/cpp/src/wsdb/wsdb_scheduler.hpp index 550f6b8edff1..c636d4647908 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_scheduler.hpp +++ b/native-packages/wsdb/cpp/src/wsdb/wsdb_scheduler.hpp @@ -26,7 +26,7 @@ * completion can safely outlive the reactor loop's return. */ -#include "barretenberg/common/thread_pool.hpp" +#include "common/thread_pool.hpp" #include "ipc_runtime/ipc_server.hpp" #include @@ -38,11 +38,11 @@ #include #include -namespace bb::wsdb { +namespace azteclabs::wsdb { class WsdbScheduler : public std::enable_shared_from_this { public: - WsdbScheduler(bb::ThreadPool& pool, ipc::IpcServer& server) + WsdbScheduler(azteclabs::wsdb::ThreadPool& pool, ipc::IpcServer& server) : pool_(pool) , server_(server) {} @@ -161,11 +161,11 @@ class WsdbScheduler : public std::enable_shared_from_this { pump(fork, lane); } - bb::ThreadPool& pool_; + azteclabs::wsdb::ThreadPool& pool_; ipc::IpcServer& server_; std::mutex mtx_; std::unordered_map lanes_; // per fork; references stable (never erased) std::atomic inflight_{ 0 }; // submitted-but-not-completed ops (queued + running) }; -} // namespace bb::wsdb +} // namespace azteclabs::wsdb diff --git a/native-packages/wsdb/cpp/src/wsdb/wsdb_wire_convert.hpp b/native-packages/wsdb/cpp/src/wsdb/wsdb_wire_convert.hpp new file mode 100644 index 000000000000..0b5e82a8d75b --- /dev/null +++ b/native-packages/wsdb/cpp/src/wsdb/wsdb_wire_convert.hpp @@ -0,0 +1,175 @@ +#pragma once +/** + * @file wsdb_wire_convert.hpp + * @brief Server-side wire <-> domain conversion helpers for the aztec-wsdb + * service. + * + * The converters that depend only on the generic merkle-tree vocabulary live + * in a local wsdb_wire_convert_client.hpp, bound to this package's generated + * wire types (bb's wsdb client keeps its own copy bound to bb's generated + * types; the two can't share one header because each side codegens its own + * wsdb_types.hpp). This header pulls those in and adds the server-only + * converters that touch world_state aggregates (state references, DB stats, + * tree/world-state meta), which require world_state and the persistent merkle + * storage. + */ +#include "merkle_tree/response.hpp" +#include "merkle_tree/tree_db_stats.hpp" +#include "merkle_tree/tree_meta.hpp" +#include "merkle_tree/types.hpp" +#include "lmdblib/types.hpp" +#include "world_state/types.hpp" +#include "world_state/world_state.hpp" +#include "wsdb/wsdb_wire_convert_client.hpp" + +namespace azteclabs::wsdb { + +inline std::vector +state_reference_to_wire(const world_state::StateReference &d) { + std::vector r; + r.reserve(d.size()); + for (const auto &[tree_id, tree_ref] : d) { + r.push_back({.treeId = tree_id_to_wire(tree_id), + .root = fr_to_wire(tree_ref.first), + .size = tree_ref.second}); + } + return r; +} + +inline world_state::StateReference +state_reference_from_wire(const std::vector &w) { + world_state::StateReference r; + r.reserve(w.size()); + for (const auto &entry : w) { + r.emplace( + tree_id_from_wire(entry.treeId), + world_state::TreeStateReference{fr_from_wire(entry.root), entry.size}); + } + return r; +} + +inline wire::DBStats db_stats_to_wire(const azteclabs::lmdblib::DBStats &d) { + return {.name = d.name, + .numDataItems = d.numDataItems, + .totalUsedSize = d.totalUsedSize}; +} + +inline azteclabs::lmdblib::DBStats db_stats_from_wire(const wire::DBStats &w) { + return azteclabs::lmdblib::DBStats(w.name, w.numDataItems, w.totalUsedSize); +} + +inline wire::TreeDBStats +tree_db_stats_to_wire(const azteclabs::wsdb::merkle_tree::TreeDBStats &d) { + return {.mapSize = d.mapSize, + .physicalFileSize = d.physicalFileSize, + .blocksDBStats = db_stats_to_wire(d.blocksDBStats), + .nodesDBStats = db_stats_to_wire(d.nodesDBStats), + .leafPreimagesDBStats = db_stats_to_wire(d.leafPreimagesDBStats), + .leafIndicesDBStats = db_stats_to_wire(d.leafIndicesDBStats), + .blockIndicesDBStats = db_stats_to_wire(d.blockIndicesDBStats)}; +} + +inline azteclabs::wsdb::merkle_tree::TreeDBStats +tree_db_stats_from_wire(const wire::TreeDBStats &w) { + return {w.mapSize, + w.physicalFileSize, + db_stats_from_wire(w.blocksDBStats), + db_stats_from_wire(w.nodesDBStats), + db_stats_from_wire(w.leafPreimagesDBStats), + db_stats_from_wire(w.leafIndicesDBStats), + db_stats_from_wire(w.blockIndicesDBStats)}; +} + +inline wire::TreeMeta +tree_meta_to_wire(const azteclabs::wsdb::merkle_tree::TreeMeta &d) { + return {.name = d.name, + .depth = d.depth, + .size = d.size, + .committedSize = d.committedSize, + .root = fr_to_wire(d.root), + .initialSize = d.initialSize, + .initialRoot = fr_to_wire(d.initialRoot), + .oldestHistoricBlock = d.oldestHistoricBlock, + .unfinalizedBlockHeight = d.unfinalizedBlockHeight, + .finalizedBlockHeight = d.finalizedBlockHeight}; +} + +inline azteclabs::wsdb::merkle_tree::TreeMeta +tree_meta_from_wire(const wire::TreeMeta &w) { + return {w.name, + w.depth, + w.size, + w.committedSize, + fr_from_wire(w.root), + w.initialSize, + fr_from_wire(w.initialRoot), + w.oldestHistoricBlock, + w.unfinalizedBlockHeight, + w.finalizedBlockHeight}; +} + +inline wire::WorldStateDBStats +world_state_db_stats_to_wire(const azteclabs::wsdb::world_state::WorldStateDBStats &d) { + return {.noteHashTreeStats = tree_db_stats_to_wire(d.noteHashTreeStats), + .messageTreeStats = tree_db_stats_to_wire(d.messageTreeStats), + .archiveTreeStats = tree_db_stats_to_wire(d.archiveTreeStats), + .publicDataTreeStats = tree_db_stats_to_wire(d.publicDataTreeStats), + .nullifierTreeStats = tree_db_stats_to_wire(d.nullifierTreeStats)}; +} + +inline azteclabs::wsdb::world_state::WorldStateDBStats +world_state_db_stats_from_wire(const wire::WorldStateDBStats &w) { + return {tree_db_stats_from_wire(w.noteHashTreeStats), + tree_db_stats_from_wire(w.messageTreeStats), + tree_db_stats_from_wire(w.archiveTreeStats), + tree_db_stats_from_wire(w.publicDataTreeStats), + tree_db_stats_from_wire(w.nullifierTreeStats)}; +} + +inline wire::WorldStateMeta +world_state_meta_to_wire(const azteclabs::wsdb::world_state::WorldStateMeta &d) { + return {.noteHashTreeMeta = tree_meta_to_wire(d.noteHashTreeMeta), + .messageTreeMeta = tree_meta_to_wire(d.messageTreeMeta), + .archiveTreeMeta = tree_meta_to_wire(d.archiveTreeMeta), + .publicDataTreeMeta = tree_meta_to_wire(d.publicDataTreeMeta), + .nullifierTreeMeta = tree_meta_to_wire(d.nullifierTreeMeta)}; +} + +inline azteclabs::wsdb::world_state::WorldStateMeta +world_state_meta_from_wire(const wire::WorldStateMeta &w) { + return {tree_meta_from_wire(w.noteHashTreeMeta), + tree_meta_from_wire(w.messageTreeMeta), + tree_meta_from_wire(w.archiveTreeMeta), + tree_meta_from_wire(w.publicDataTreeMeta), + tree_meta_from_wire(w.nullifierTreeMeta)}; +} + +inline wire::WorldStateStatusSummary world_state_status_summary_to_wire( + const azteclabs::wsdb::world_state::WorldStateStatusSummary &d) { + return {.unfinalizedBlockNumber = d.unfinalizedBlockNumber, + .finalizedBlockNumber = d.finalizedBlockNumber, + .oldestHistoricalBlock = d.oldestHistoricalBlock, + .treesAreSynched = d.treesAreSynched}; +} + +inline azteclabs::wsdb::world_state::WorldStateStatusSummary +world_state_status_summary_from_wire(const wire::WorldStateStatusSummary &w) { + return {w.unfinalizedBlockNumber, w.finalizedBlockNumber, + w.oldestHistoricalBlock, w.treesAreSynched}; +} + +inline wire::WorldStateStatusFull world_state_status_full_to_wire( + const azteclabs::wsdb::world_state::WorldStateStatusFull &d) { + return {.summary = world_state_status_summary_to_wire(d.summary), + .dbStats = world_state_db_stats_to_wire(d.dbStats), + .meta = world_state_meta_to_wire(d.meta)}; +} + +inline azteclabs::wsdb::world_state::WorldStateStatusFull +world_state_status_full_from_wire(const wire::WorldStateStatusFull &w) { + return {world_state_status_summary_from_wire(w.summary), + world_state_db_stats_from_wire(w.dbStats), + world_state_meta_from_wire(w.meta)}; +} + +} // namespace azteclabs::wsdb diff --git a/native-packages/wsdb/cpp/src/wsdb/wsdb_wire_convert_client.hpp b/native-packages/wsdb/cpp/src/wsdb/wsdb_wire_convert_client.hpp new file mode 100644 index 000000000000..2fa0a8350830 --- /dev/null +++ b/native-packages/wsdb/cpp/src/wsdb/wsdb_wire_convert_client.hpp @@ -0,0 +1,360 @@ +#pragma once +/** + * @file wsdb_wire_convert.hpp + * @brief Client-side wire <-> domain conversion helpers for the aztec-wsdb service. + * + * These converters depend only on the generic merkle-tree vocabulary + * (crypto/merkle_tree) plus the generated wsdb wire types, so they can be + * compiled into bb's wsdb client (consumed by the AVM simulator) without + * pulling in world_state or the persistent merkle storage. The server-side + * converters that touch world_state aggregates (state references, DB stats, + * tree/world-state meta) live alongside the server in native-packages/wsdb. + */ +#include "field/field_element.hpp" +#include "merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/merkle_tree_id.hpp" +#include "merkle_tree/response.hpp" +#include "merkle_tree/types.hpp" +#include "wsdb/generated/wsdb_types.hpp" + +namespace azteclabs::wsdb { + +inline wire::Fr fr_to_wire(const azteclabs::wsdb::fr& d) +{ + wire::Fr r{}; + azteclabs::wsdb::fr::serialize_to_buffer(d, r.data()); + return r; +} + +inline azteclabs::wsdb::fr fr_from_wire(const wire::Fr& w) +{ + return azteclabs::wsdb::fr::serialize_from_buffer(w.data()); +} + +inline wire::BlockHeaderHash block_header_hash_to_wire(const azteclabs::wsdb::fr& d) +{ + wire::BlockHeaderHash r{}; + azteclabs::wsdb::fr::serialize_to_buffer(d, r.data()); + return r; +} + +inline azteclabs::wsdb::fr block_header_hash_from_wire(const wire::BlockHeaderHash& w) +{ + return azteclabs::wsdb::fr::serialize_from_buffer(w.data()); +} + +inline wire::PublicDataSlot public_data_slot_to_wire(const azteclabs::wsdb::fr& d) +{ + wire::PublicDataSlot r{}; + azteclabs::wsdb::fr::serialize_to_buffer(d, r.data()); + return r; +} + +inline azteclabs::wsdb::fr public_data_slot_from_wire(const wire::PublicDataSlot& w) +{ + return azteclabs::wsdb::fr::serialize_from_buffer(w.data()); +} + +inline wire::PublicDataValue public_data_value_to_wire(const azteclabs::wsdb::fr& d) +{ + wire::PublicDataValue r{}; + azteclabs::wsdb::fr::serialize_to_buffer(d, r.data()); + return r; +} + +inline azteclabs::wsdb::fr public_data_value_from_wire(const wire::PublicDataValue& w) +{ + return azteclabs::wsdb::fr::serialize_from_buffer(w.data()); +} + +inline wire::Nullifier nullifier_to_wire(const azteclabs::wsdb::fr& d) +{ + wire::Nullifier r{}; + azteclabs::wsdb::fr::serialize_to_buffer(d, r.data()); + return r; +} + +inline azteclabs::wsdb::fr nullifier_from_wire(const wire::Nullifier& w) +{ + return azteclabs::wsdb::fr::serialize_from_buffer(w.data()); +} + +inline std::vector fr_vec_to_wire(const std::vector& d) +{ + std::vector r; + r.reserve(d.size()); + for (const auto& x : d) { + r.push_back(fr_to_wire(x)); + } + return r; +} + +inline std::vector fr_vec_from_wire(const std::vector& w) +{ + std::vector r; + r.reserve(w.size()); + for (const auto& x : w) { + r.push_back(fr_from_wire(x)); + } + return r; +} + +inline wire::WorldStateRevision revision_to_wire(const merkle_tree::WorldStateRevision& d) +{ + return wire::WorldStateRevision{ + .forkId = d.forkId, + .blockNumber = d.blockNumber, + .includeUncommitted = d.includeUncommitted, + }; +} + +inline merkle_tree::WorldStateRevision revision_from_wire(const wire::WorldStateRevision& w) +{ + return merkle_tree::WorldStateRevision{ + .forkId = w.forkId, + .blockNumber = w.blockNumber, + .includeUncommitted = w.includeUncommitted, + }; +} + +inline wire::MerkleTreeId tree_id_to_wire(merkle_tree::MerkleTreeId d) +{ + return static_cast(d); +} + +inline merkle_tree::MerkleTreeId tree_id_from_wire(wire::MerkleTreeId w) +{ + return static_cast(w); +} + +inline wire::PublicDataLeafValue public_data_leaf_to_wire(const merkle_tree::PublicDataLeafValue& d) +{ + return { .slot = public_data_slot_to_wire(d.slot), .value = public_data_value_to_wire(d.value) }; +} + +inline merkle_tree::PublicDataLeafValue public_data_leaf_from_wire(const wire::PublicDataLeafValue& w) +{ + return { public_data_slot_from_wire(w.slot), public_data_value_from_wire(w.value) }; +} + +inline std::vector public_data_leaf_vec_to_wire( + const std::vector& d) +{ + std::vector r; + r.reserve(d.size()); + for (const auto& x : d) { + r.push_back(public_data_leaf_to_wire(x)); + } + return r; +} + +inline std::vector public_data_leaf_vec_from_wire( + const std::vector& w) +{ + std::vector r; + r.reserve(w.size()); + for (const auto& x : w) { + r.push_back(public_data_leaf_from_wire(x)); + } + return r; +} + +inline wire::NullifierLeafValue nullifier_leaf_to_wire(const merkle_tree::NullifierLeafValue& d) +{ + return { .nullifier = nullifier_to_wire(d.nullifier) }; +} + +inline merkle_tree::NullifierLeafValue nullifier_leaf_from_wire(const wire::NullifierLeafValue& w) +{ + return { nullifier_from_wire(w.nullifier) }; +} + +inline std::vector nullifier_leaf_vec_to_wire( + const std::vector& d) +{ + std::vector r; + r.reserve(d.size()); + for (const auto& x : d) { + r.push_back(nullifier_leaf_to_wire(x)); + } + return r; +} + +inline std::vector nullifier_leaf_vec_from_wire( + const std::vector& w) +{ + std::vector r; + r.reserve(w.size()); + for (const auto& x : w) { + r.push_back(nullifier_leaf_from_wire(x)); + } + return r; +} + +inline wire::IndexedPublicDataLeafValue indexed_public_data_leaf_to_wire( + const merkle_tree::IndexedLeaf& d) +{ + return { .leaf = public_data_leaf_to_wire(d.leaf), .nextIndex = d.nextIndex, .nextKey = fr_to_wire(d.nextKey) }; +} + +inline merkle_tree::IndexedLeaf indexed_public_data_leaf_from_wire( + const wire::IndexedPublicDataLeafValue& w) +{ + return { public_data_leaf_from_wire(w.leaf), w.nextIndex, fr_from_wire(w.nextKey) }; +} + +inline wire::IndexedNullifierLeafValue indexed_nullifier_leaf_to_wire( + const merkle_tree::IndexedLeaf& d) +{ + return { .leaf = nullifier_leaf_to_wire(d.leaf), .nextIndex = d.nextIndex, .nextKey = fr_to_wire(d.nextKey) }; +} + +inline merkle_tree::IndexedLeaf indexed_nullifier_leaf_from_wire( + const wire::IndexedNullifierLeafValue& w) +{ + return { nullifier_leaf_from_wire(w.leaf), w.nextIndex, fr_from_wire(w.nextKey) }; +} + +inline wire::PublicDataLeafUpdateWitnessData public_data_witness_to_wire( + const merkle_tree::LeafUpdateWitnessData& d) +{ + return { .leaf = indexed_public_data_leaf_to_wire(d.leaf), .index = d.index, .path = fr_vec_to_wire(d.path) }; +} + +inline merkle_tree::LeafUpdateWitnessData public_data_witness_from_wire( + const wire::PublicDataLeafUpdateWitnessData& w) +{ + return { indexed_public_data_leaf_from_wire(w.leaf), w.index, fr_vec_from_wire(w.path) }; +} + +inline wire::NullifierLeafUpdateWitnessData nullifier_witness_to_wire( + const merkle_tree::LeafUpdateWitnessData& d) +{ + return { .leaf = indexed_nullifier_leaf_to_wire(d.leaf), .index = d.index, .path = fr_vec_to_wire(d.path) }; +} + +inline merkle_tree::LeafUpdateWitnessData nullifier_witness_from_wire( + const wire::NullifierLeafUpdateWitnessData& w) +{ + return { indexed_nullifier_leaf_from_wire(w.leaf), w.index, fr_vec_from_wire(w.path) }; +} + +template +inline std::vector vec_to_wire(const std::vector& d, Fn fn) +{ + std::vector r; + r.reserve(d.size()); + for (const auto& x : d) { + r.push_back(fn(x)); + } + return r; +} + +template +inline std::vector vec_from_wire(const std::vector& w, Fn fn) +{ + std::vector r; + r.reserve(w.size()); + for (const auto& x : w) { + r.push_back(fn(x)); + } + return r; +} + +inline wire::BatchInsertionResultPublicData batch_public_data_to_wire( + const merkle_tree::BatchInsertionResult& d) +{ + std::vector sorted; + sorted.reserve(d.sorted_leaves.size()); + for (const auto& [leaf, index] : d.sorted_leaves) { + sorted.push_back({ .leaf = public_data_leaf_to_wire(leaf), .index = index }); + } + return { .lowLeafWitnessData = vec_to_wire(d.low_leaf_witness_data, + public_data_witness_to_wire), + .sortedLeaves = std::move(sorted), + .subtreePath = fr_vec_to_wire(d.subtree_path) }; +} + +inline merkle_tree::BatchInsertionResult batch_public_data_from_wire( + const wire::BatchInsertionResultPublicData& w) +{ + merkle_tree::BatchInsertionResult r; + r.low_leaf_witness_data = vec_from_wire>( + w.lowLeafWitnessData, public_data_witness_from_wire); + r.sorted_leaves.reserve(w.sortedLeaves.size()); + for (const auto& x : w.sortedLeaves) { + r.sorted_leaves.emplace_back(public_data_leaf_from_wire(x.leaf), x.index); + } + r.subtree_path = fr_vec_from_wire(w.subtreePath); + return r; +} + +inline wire::BatchInsertionResultNullifier batch_nullifier_to_wire( + const merkle_tree::BatchInsertionResult& d) +{ + std::vector sorted; + sorted.reserve(d.sorted_leaves.size()); + for (const auto& [leaf, index] : d.sorted_leaves) { + sorted.push_back({ .leaf = nullifier_leaf_to_wire(leaf), .index = index }); + } + return { .lowLeafWitnessData = + vec_to_wire(d.low_leaf_witness_data, nullifier_witness_to_wire), + .sortedLeaves = std::move(sorted), + .subtreePath = fr_vec_to_wire(d.subtree_path) }; +} + +inline merkle_tree::BatchInsertionResult batch_nullifier_from_wire( + const wire::BatchInsertionResultNullifier& w) +{ + merkle_tree::BatchInsertionResult r; + r.low_leaf_witness_data = vec_from_wire>( + w.lowLeafWitnessData, nullifier_witness_from_wire); + r.sorted_leaves.reserve(w.sortedLeaves.size()); + for (const auto& x : w.sortedLeaves) { + r.sorted_leaves.emplace_back(nullifier_leaf_from_wire(x.leaf), x.index); + } + r.subtree_path = fr_vec_from_wire(w.subtreePath); + return r; +} + +inline wire::SequentialInsertionResultPublicData sequential_public_data_to_wire( + const merkle_tree::SequentialInsertionResult& d) +{ + return { .lowLeafWitnessData = vec_to_wire(d.low_leaf_witness_data, + public_data_witness_to_wire), + .insertionWitnessData = vec_to_wire(d.insertion_witness_data, + public_data_witness_to_wire) }; +} + +inline merkle_tree::SequentialInsertionResult sequential_public_data_from_wire( + const wire::SequentialInsertionResultPublicData& w) +{ + return { .low_leaf_witness_data = + vec_from_wire>( + w.lowLeafWitnessData, public_data_witness_from_wire), + .insertion_witness_data = + vec_from_wire>( + w.insertionWitnessData, public_data_witness_from_wire) }; +} + +inline wire::SequentialInsertionResultNullifier sequential_nullifier_to_wire( + const merkle_tree::SequentialInsertionResult& d) +{ + return { .lowLeafWitnessData = + vec_to_wire(d.low_leaf_witness_data, nullifier_witness_to_wire), + .insertionWitnessData = vec_to_wire(d.insertion_witness_data, + nullifier_witness_to_wire) }; +} + +inline merkle_tree::SequentialInsertionResult sequential_nullifier_from_wire( + const wire::SequentialInsertionResultNullifier& w) +{ + return { .low_leaf_witness_data = + vec_from_wire>( + w.lowLeafWitnessData, nullifier_witness_from_wire), + .insertion_witness_data = + vec_from_wire>( + w.insertionWitnessData, nullifier_witness_from_wire) }; +} + +} // namespace azteclabs::wsdb diff --git a/wsdb/package.json b/native-packages/wsdb/package.json similarity index 84% rename from wsdb/package.json rename to native-packages/wsdb/package.json index d2a88a1534f9..cb84a9543890 100644 --- a/wsdb/package.json +++ b/native-packages/wsdb/package.json @@ -11,6 +11,6 @@ "ts/packages/*" ], "resolutions": { - "@aztec/ipc-runtime": "portal:../ipc-runtime/ts" + "@aztec/ipc-runtime": "portal:../../ipc-runtime/ts" } } diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_schema.jsonc b/native-packages/wsdb/wsdb_schema.jsonc similarity index 100% rename from barretenberg/cpp/src/barretenberg/wsdb/wsdb_schema.jsonc rename to native-packages/wsdb/wsdb_schema.jsonc diff --git a/wsdb/yarn.lock b/native-packages/wsdb/yarn.lock similarity index 98% rename from wsdb/yarn.lock rename to native-packages/wsdb/yarn.lock index 22dfff672f55..d0f2cc8d4e7d 100644 --- a/wsdb/yarn.lock +++ b/native-packages/wsdb/yarn.lock @@ -5,9 +5,9 @@ __metadata: version: 8 cacheKey: 10c0 -"@aztec/ipc-runtime@portal:../ipc-runtime/ts::locator=%40aztec%2Fwsdb-packages%40workspace%3A.": +"@aztec/ipc-runtime@portal:../../ipc-runtime/ts::locator=%40aztec%2Fwsdb-packages%40workspace%3A.": version: 0.0.0-use.local - resolution: "@aztec/ipc-runtime@portal:../ipc-runtime/ts::locator=%40aztec%2Fwsdb-packages%40workspace%3A." + resolution: "@aztec/ipc-runtime@portal:../../ipc-runtime/ts::locator=%40aztec%2Fwsdb-packages%40workspace%3A." languageName: node linkType: soft diff --git a/release-image/Dockerfile.base.dockerignore b/release-image/Dockerfile.base.dockerignore index b81140f52f7b..b95912754322 100644 --- a/release-image/Dockerfile.base.dockerignore +++ b/release-image/Dockerfile.base.dockerignore @@ -4,8 +4,10 @@ !/barretenberg/ts/bb-avm-sim/package.json !/barretenberg/ts/bb-avm-sim/packages/*/package.json !/noir/packages/*/package.json -!/wsdb/ts/package.json -!/wsdb/ts/packages/*/package.json +!/native-packages/wsdb/ts/package.json +!/native-packages/wsdb/ts/packages/*/package.json +!/native-packages/kvdb/ts/package.json +!/native-packages/kvdb/ts/packages/*/package.json !/ipc-runtime/ts/package.json !/yarn-project/package.json !/yarn-project/yarn.lock diff --git a/release-image/Dockerfile.dockerignore b/release-image/Dockerfile.dockerignore index 573a36a3c80c..495c744e35e5 100644 --- a/release-image/Dockerfile.dockerignore +++ b/release-image/Dockerfile.dockerignore @@ -11,10 +11,14 @@ !/barretenberg/ts/bb-avm-sim/packages/ !/ipc-runtime/ts/dest/ !/ipc-runtime/ts/package.json -!/wsdb/ts/dest/ -!/wsdb/ts/build/ -!/wsdb/ts/package.json -!/wsdb/ts/packages/ +!/native-packages/wsdb/ts/dest/ +!/native-packages/wsdb/ts/build/ +!/native-packages/wsdb/ts/package.json +!/native-packages/wsdb/ts/packages/ +!/native-packages/kvdb/ts/dest/ +!/native-packages/kvdb/ts/build/ +!/native-packages/kvdb/ts/package.json +!/native-packages/kvdb/ts/packages/ !/noir/noir-repo/target/release/nargo !/noir/noir-repo/target/release/acvm !/noir/packages/ diff --git a/release-image/bootstrap.sh b/release-image/bootstrap.sh index 5a25ff0a099f..7b1d0dc47e16 100755 --- a/release-image/bootstrap.sh +++ b/release-image/bootstrap.sh @@ -3,7 +3,8 @@ source $(git rev-parse --show-toplevel)/ci3/source_bootstrap hash=$(hash_str \ $(cache_content_hash ^release-image/Dockerfile ^release-image/Dockerfile.base.dockerignore ^release-image/Dockerfile.dockerignore ^build-images/src/Dockerfile ^ipc-runtime/ts/package.json ^yarn-project/yarn.lock) \ - $(../wsdb/bootstrap.sh hash) \ + $(../native-packages/kvdb/bootstrap.sh hash) \ + $(../native-packages/wsdb/bootstrap.sh hash) \ $(../barretenberg/ts/bootstrap.sh hash)) function prepare_crs { diff --git a/wsdb/.rebuild_patterns b/wsdb/.rebuild_patterns deleted file mode 100644 index 1652d3c01a63..000000000000 --- a/wsdb/.rebuild_patterns +++ /dev/null @@ -1,6 +0,0 @@ -^wsdb/bootstrap\.sh$ -^wsdb/package\.json$ -^wsdb/\.rebuild_patterns$ -^ipc-codegen/src/.*\.ts$ -^ipc-codegen/templates/ -^barretenberg/cpp/src/barretenberg/wsdb/wsdb_schema\.json$ diff --git a/wsdb/bootstrap.sh b/wsdb/bootstrap.sh deleted file mode 100755 index 577cc1e5779e..000000000000 --- a/wsdb/bootstrap.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash -source $(git rev-parse --show-toplevel)/ci3/source_bootstrap - -ROOT=$(git rev-parse --show-toplevel) -WSDB_BINARY=aztec-wsdb - -hash=$(hash_str \ - $(../barretenberg/cpp/bootstrap.sh hash) \ - $(../ipc-runtime/bootstrap.sh hash) \ - $(cache_content_hash .rebuild_patterns)) - -function generate_ts_package { - node --experimental-strip-types --experimental-transform-types --no-warnings \ - "$ROOT/ipc-codegen/src/generate.ts" \ - --schema "$ROOT/barretenberg/cpp/src/barretenberg/wsdb/wsdb_schema.jsonc" \ - --lang ts \ - --client \ - --out "$ROOT/wsdb/ts/src/generated" \ - --package "$ROOT/wsdb/ts" \ - --package-name @aztec/wsdb \ - --binary-name "$WSDB_BINARY" \ - --package-transports uds,shm \ - --package-ipc-path-args 'msgpack,run,--input,{path}' -} - -function copy_native { - local target_dir="ts/build/$(arch)-$(os)" - mkdir -p "$target_dir" - cp "$ROOT/barretenberg/cpp/build/bin/$WSDB_BINARY" "$target_dir/$WSDB_BINARY" -} - -function copy_cross { - if [ -n "${1:-}" ]; then - local cross_arch="$1" - mkdir -p "ts/build/$cross_arch" - cp "$ROOT/barretenberg/cpp/build-$cross_arch/bin/$WSDB_BINARY" "ts/build/$cross_arch/$WSDB_BINARY" - elif semver check "${REF_NAME:-}" && [ "$(arch)" == "amd64" ]; then - for cross_arch in arm64-linux amd64-macos arm64-macos; do - mkdir -p "ts/build/$cross_arch" - cp "$ROOT/barretenberg/cpp/build-$cross_arch/bin/$WSDB_BINARY" "ts/build/$cross_arch/$WSDB_BINARY" - done - else - echo "This task is expected to be run with an explicit arch or in an x86 release context." - fi -} - -function build { - echo_header "wsdb build" - generate_ts_package - copy_native - npm_install_deps - yarn build - (cd ts && ./scripts/prepare_arch_packages.sh "$(arch)-$(os)=build/$(arch)-$(os)/$WSDB_BINARY") -} - -function clean { - rm -rf ts node_modules -} - -function release { - generate_ts_package - copy_native - copy_cross - npm_install_deps - yarn build - (cd ts && ./scripts/prepare_arch_packages.sh) - for package_dir in ts/packages/*; do - (cd "$package_dir" && retry "deploy_npm ${REF_NAME#v}") - done - (cd ts && retry "deploy_npm ${REF_NAME#v}") -} - -export -f generate_ts_package copy_native copy_cross build clean release - -case "$cmd" in - "") - build - ;; - "hash") - echo "$hash" - ;; - *) - default_cmd_handler "$@" - ;; -esac diff --git a/yarn-project/constants/src/scripts/constants.in.ts b/yarn-project/constants/src/scripts/constants.in.ts index c5df26e3385f..e99be4a20d14 100644 --- a/yarn-project/constants/src/scripts/constants.in.ts +++ b/yarn-project/constants/src/scripts/constants.in.ts @@ -5,6 +5,9 @@ import { fileURLToPath } from 'url'; const NOIR_CONSTANTS_FILE = '../../../../noir-projects/noir-protocol-circuits/crates/types/src/constants.nr'; const TS_CONSTANTS_FILE = '../constants.gen.ts'; const CPP_AZTEC_CONSTANTS_FILE = '../../../../barretenberg/cpp/src/barretenberg/aztec/aztec_constants.hpp'; +// The wsdb native package is barretenberg-header-free, so it keeps its own copy of the same +// generated constants rather than including bb's. Regenerated here to stay in lockstep. +const WSDB_CPP_AZTEC_CONSTANTS_FILE = '../../../../native-packages/wsdb/cpp/src/common/aztec_constants.hpp'; const PIL_AZTEC_CONSTANTS_FILE = '../../../../barretenberg/cpp/pil/vm2/constants_gen.pil'; const SOLIDITY_CONSTANTS_FILE = '../../../../l1-contracts/src/core/libraries/ConstantsGen.sol'; @@ -729,6 +732,10 @@ function main(): void { const cppTargetPath = join(__dirname, CPP_AZTEC_CONSTANTS_FILE); generateCppConstants(parsedContent, cppTargetPath); + // Cpp (wsdb native package's bb-header-free copy — identical content) + const wsdbCppTargetPath = join(__dirname, WSDB_CPP_AZTEC_CONSTANTS_FILE); + generateCppConstants(parsedContent, wsdbCppTargetPath); + // PIL const pilTargetPath = join(__dirname, PIL_AZTEC_CONSTANTS_FILE); generatePilConstants(parsedContent, pilTargetPath); diff --git a/yarn-project/native/package.json b/yarn-project/native/package.json index 4c0fb4df0f58..8583fcb6fc9b 100644 --- a/yarn-project/native/package.json +++ b/yarn-project/native/package.json @@ -17,6 +17,7 @@ "dependencies": { "@aztec/bb.js": "workspace:^", "@aztec/foundation": "workspace:^", + "@aztec/kvdb": "0.1.0", "msgpackr": "^1.11.2" }, "devDependencies": { diff --git a/yarn-project/native/src/native_module.ts b/yarn-project/native/src/native_module.ts index 25d477906e35..2bf94be62ee3 100644 --- a/yarn-project/native/src/native_module.ts +++ b/yarn-project/native/src/native_module.ts @@ -1,4 +1,4 @@ -import { findNapiBinary } from '@aztec/bb.js'; +import { findNapiBinary } from '@aztec/kvdb'; import { createRequire } from 'module'; diff --git a/yarn-project/package.json b/yarn-project/package.json index 4769224cc8ac..cd9d2a443384 100644 --- a/yarn-project/package.json +++ b/yarn-project/package.json @@ -91,11 +91,16 @@ "@aztec/bb-avm-sim-linux-x64": "portal:../barretenberg/ts/bb-avm-sim/packages/bb-avm-sim-linux-x64", "@aztec/bb.js": "portal:../barretenberg/ts/bb.js", "@aztec/ipc-runtime": "portal:../ipc-runtime/ts", - "@aztec/wsdb": "portal:../wsdb/ts", - "@aztec/wsdb-darwin-arm64": "portal:../wsdb/ts/packages/wsdb-darwin-arm64", - "@aztec/wsdb-darwin-x64": "portal:../wsdb/ts/packages/wsdb-darwin-x64", - "@aztec/wsdb-linux-arm64": "portal:../wsdb/ts/packages/wsdb-linux-arm64", - "@aztec/wsdb-linux-x64": "portal:../wsdb/ts/packages/wsdb-linux-x64", + "@aztec/wsdb": "portal:../native-packages/wsdb/ts", + "@aztec/wsdb-darwin-arm64": "portal:../native-packages/wsdb/ts/packages/wsdb-darwin-arm64", + "@aztec/wsdb-darwin-x64": "portal:../native-packages/wsdb/ts/packages/wsdb-darwin-x64", + "@aztec/wsdb-linux-arm64": "portal:../native-packages/wsdb/ts/packages/wsdb-linux-arm64", + "@aztec/wsdb-linux-x64": "portal:../native-packages/wsdb/ts/packages/wsdb-linux-x64", + "@aztec/kvdb": "portal:../native-packages/kvdb/ts", + "@aztec/kvdb-darwin-arm64": "portal:../native-packages/kvdb/ts/packages/kvdb-darwin-arm64", + "@aztec/kvdb-darwin-x64": "portal:../native-packages/kvdb/ts/packages/kvdb-darwin-x64", + "@aztec/kvdb-linux-arm64": "portal:../native-packages/kvdb/ts/packages/kvdb-linux-arm64", + "@aztec/kvdb-linux-x64": "portal:../native-packages/kvdb/ts/packages/kvdb-linux-x64", "@aztec/noir-acvm_js": "portal:../noir/packages/acvm_js", "@aztec/noir-types": "portal:../noir/packages/types", "@aztec/noir-noirc_abi": "portal:../noir/packages/noirc_abi", diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 47e8be717696..261e8044e1b0 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -1567,6 +1567,55 @@ __metadata: languageName: unknown linkType: soft +"@aztec/kvdb-darwin-arm64@portal:../native-packages/kvdb/ts/packages/kvdb-darwin-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": + version: 0.0.0-use.local + resolution: "@aztec/kvdb-darwin-arm64@portal:../native-packages/kvdb/ts/packages/kvdb-darwin-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A." + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: soft + +"@aztec/kvdb-darwin-x64@portal:../native-packages/kvdb/ts/packages/kvdb-darwin-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": + version: 0.0.0-use.local + resolution: "@aztec/kvdb-darwin-x64@portal:../native-packages/kvdb/ts/packages/kvdb-darwin-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A." + conditions: os=darwin & cpu=x64 + languageName: node + linkType: soft + +"@aztec/kvdb-linux-arm64@portal:../native-packages/kvdb/ts/packages/kvdb-linux-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": + version: 0.0.0-use.local + resolution: "@aztec/kvdb-linux-arm64@portal:../native-packages/kvdb/ts/packages/kvdb-linux-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A." + conditions: os=linux & cpu=arm64 + languageName: node + linkType: soft + +"@aztec/kvdb-linux-x64@portal:../native-packages/kvdb/ts/packages/kvdb-linux-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": + version: 0.0.0-use.local + resolution: "@aztec/kvdb-linux-x64@portal:../native-packages/kvdb/ts/packages/kvdb-linux-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A." + conditions: os=linux & cpu=x64 + languageName: node + linkType: soft + +"@aztec/kvdb@portal:../native-packages/kvdb/ts::locator=%40aztec%2Faztec3-packages%40workspace%3A.": + version: 0.0.0-use.local + resolution: "@aztec/kvdb@portal:../native-packages/kvdb/ts::locator=%40aztec%2Faztec3-packages%40workspace%3A." + dependencies: + "@aztec/kvdb-darwin-arm64": "npm:0.1.0" + "@aztec/kvdb-darwin-x64": "npm:0.1.0" + "@aztec/kvdb-linux-arm64": "npm:0.1.0" + "@aztec/kvdb-linux-x64": "npm:0.1.0" + tslib: "npm:^2.4.0" + dependenciesMeta: + "@aztec/kvdb-darwin-arm64": + optional: true + "@aztec/kvdb-darwin-x64": + optional: true + "@aztec/kvdb-linux-arm64": + optional: true + "@aztec/kvdb-linux-x64": + optional: true + languageName: node + linkType: soft + "@aztec/l1-artifacts@portal:../../l1-contracts/l1-artifacts::locator=%40aztec%2Farchiver%40workspace%3Aarchiver": version: 0.0.0-use.local resolution: "@aztec/l1-artifacts@portal:../../l1-contracts/l1-artifacts::locator=%40aztec%2Farchiver%40workspace%3Aarchiver" @@ -1677,6 +1726,7 @@ __metadata: dependencies: "@aztec/bb.js": "workspace:^" "@aztec/foundation": "workspace:^" + "@aztec/kvdb": "npm:0.1.0" "@jest/globals": "npm:^30.0.0" "@types/jest": "npm:^30.0.0" "@types/node": "npm:^22.15.17" @@ -2503,37 +2553,37 @@ __metadata: languageName: unknown linkType: soft -"@aztec/wsdb-darwin-arm64@portal:../wsdb/ts/packages/wsdb-darwin-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": +"@aztec/wsdb-darwin-arm64@portal:../native-packages/wsdb/ts/packages/wsdb-darwin-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": version: 0.0.0-use.local - resolution: "@aztec/wsdb-darwin-arm64@portal:../wsdb/ts/packages/wsdb-darwin-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A." + resolution: "@aztec/wsdb-darwin-arm64@portal:../native-packages/wsdb/ts/packages/wsdb-darwin-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A." conditions: os=darwin & cpu=arm64 languageName: node linkType: soft -"@aztec/wsdb-darwin-x64@portal:../wsdb/ts/packages/wsdb-darwin-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": +"@aztec/wsdb-darwin-x64@portal:../native-packages/wsdb/ts/packages/wsdb-darwin-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": version: 0.0.0-use.local - resolution: "@aztec/wsdb-darwin-x64@portal:../wsdb/ts/packages/wsdb-darwin-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A." + resolution: "@aztec/wsdb-darwin-x64@portal:../native-packages/wsdb/ts/packages/wsdb-darwin-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A." conditions: os=darwin & cpu=x64 languageName: node linkType: soft -"@aztec/wsdb-linux-arm64@portal:../wsdb/ts/packages/wsdb-linux-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": +"@aztec/wsdb-linux-arm64@portal:../native-packages/wsdb/ts/packages/wsdb-linux-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": version: 0.0.0-use.local - resolution: "@aztec/wsdb-linux-arm64@portal:../wsdb/ts/packages/wsdb-linux-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A." + resolution: "@aztec/wsdb-linux-arm64@portal:../native-packages/wsdb/ts/packages/wsdb-linux-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A." conditions: os=linux & cpu=arm64 languageName: node linkType: soft -"@aztec/wsdb-linux-x64@portal:../wsdb/ts/packages/wsdb-linux-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": +"@aztec/wsdb-linux-x64@portal:../native-packages/wsdb/ts/packages/wsdb-linux-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": version: 0.0.0-use.local - resolution: "@aztec/wsdb-linux-x64@portal:../wsdb/ts/packages/wsdb-linux-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A." + resolution: "@aztec/wsdb-linux-x64@portal:../native-packages/wsdb/ts/packages/wsdb-linux-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A." conditions: os=linux & cpu=x64 languageName: node linkType: soft -"@aztec/wsdb@portal:../wsdb/ts::locator=%40aztec%2Faztec3-packages%40workspace%3A.": +"@aztec/wsdb@portal:../native-packages/wsdb/ts::locator=%40aztec%2Faztec3-packages%40workspace%3A.": version: 0.0.0-use.local - resolution: "@aztec/wsdb@portal:../wsdb/ts::locator=%40aztec%2Faztec3-packages%40workspace%3A." + resolution: "@aztec/wsdb@portal:../native-packages/wsdb/ts::locator=%40aztec%2Faztec3-packages%40workspace%3A." dependencies: "@aztec/ipc-runtime": "@aztec/ipc-runtime" "@aztec/wsdb-darwin-arm64": "npm:0.1.0"