diff --git a/Makefile b/Makefile index 5b8a527bc79c..1d10b9cef1f9 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ endef # PHONY TARGETS - List every target that has a file/dir of the same name. #============================================================================== -.PHONY: noir barretenberg noir-projects l1-contracts release-image boxes playground docs aztec-up spartan wsdb bb-avm-sim +.PHONY: noir barretenberg noir-projects l1-contracts release-image boxes playground docs aztec-up spartan lmdblib kvdb wsdb bb-avm-sim #============================================================================== # BOOTSTRAP TARGETS @@ -321,11 +321,20 @@ ipc-runtime-cross-arm64-macos: ipc-runtime-cross: ipc-runtime ipc-runtime-cross-arm64-linux ipc-runtime-cross-amd64-macos ipc-runtime-cross-arm64-macos #============================================================================== -# WSDB +# Native packages (lmdblib, kvdb, wsdb) #============================================================================== -wsdb: ipc-codegen ipc-runtime bb-cpp-native - $(call build,$@,wsdb) +# lmdblib and kvdb are barretenberg-free: they build against their own deps +# (lmdb, msgpack-c, node-addon-api) only, never bb. +.PHONY: lmdblib kvdb +lmdblib: + $(call build,$@,native-packages/lmdblib) + +kvdb: lmdblib + $(call build,$@,native-packages/kvdb) + +wsdb: ipc-codegen ipc-runtime bb-cpp-native lmdblib + $(call build,$@,native-packages/wsdb) #============================================================================== # .claude tooling @@ -393,7 +402,7 @@ l1-contracts-tests: l1-contracts-verifier # Yarn Project - TypeScript monorepo with all TS packages #============================================================================== -yarn-project: bb-ts noir-projects l1-contracts wsdb bb-avm-sim +yarn-project: bb-ts noir-projects l1-contracts wsdb kvdb bb-avm-sim $(call build,$@,yarn-project) yarn-project-tests: yarn-project diff --git a/aztec-up/bootstrap.sh b/aztec-up/bootstrap.sh index a9c383ad9329..aa9e3c9413e6 100755 --- a/aztec-up/bootstrap.sh +++ b/aztec-up/bootstrap.sh @@ -1,16 +1,23 @@ #!/usr/bin/env bash source $(git rev-parse --show-toplevel)/ci3/source_bootstrap -hash=$(hash_str $(cache_content_hash ^aztec-up/) $(../ipc-runtime/bootstrap.sh hash) $(../wsdb/bootstrap.sh hash) $(../barretenberg/ts/bootstrap.sh hash) $(../yarn-project/bootstrap.sh hash)) +hash=$(hash_str $(cache_content_hash ^aztec-up/) $(../ipc-runtime/bootstrap.sh hash) $(../native-packages/kvdb/bootstrap.sh hash) $(../native-packages/wsdb/bootstrap.sh hash) $(../barretenberg/ts/bootstrap.sh hash) $(../yarn-project/bootstrap.sh hash)) # Bare aliases ("nightly", "latest") resolve to this major version. DEFAULT_MAJOR_VERSION=${AZTEC_TOOLCHAIN_DEFAULT_MAJOR_VERSION:-4} function wsdb_package_dirs { - for package_dir in "$root"/wsdb/ts/packages/*; do + for package_dir in "$root"/native-packages/wsdb/ts/packages/*; do [ -d "$package_dir" ] && echo "$package_dir" done - echo "$root/wsdb/ts" + echo "$root/native-packages/wsdb/ts" +} + +function kvdb_package_dirs { + for package_dir in "$root"/native-packages/kvdb/ts/packages/*; do + [ -d "$package_dir" ] && echo "$package_dir" + done + echo "$root/native-packages/kvdb/ts" } function barretenberg_ts_package_dirs { @@ -117,6 +124,7 @@ EOF echo $root/ipc-runtime/ts barretenberg_ts_package_dirs wsdb_package_dirs + kvdb_package_dirs $root/noir/bootstrap.sh get_projects $root/yarn-project/bootstrap.sh get_projects } | DRY_RUN= parallel --tag --line-buffer --halt now,fail=1 "retry 'cd {} && dump_fail \"deploy_npm $version\" >/dev/null'" diff --git a/barretenberg/cpp/CMakeLists.txt b/barretenberg/cpp/CMakeLists.txt index a25b022c0448..4260bbdbcdce 100644 --- a/barretenberg/cpp/CMakeLists.txt +++ b/barretenberg/cpp/CMakeLists.txt @@ -12,7 +12,7 @@ project( # === Platform Guard Reference === # __wasm__ : Compiler-defined for WebAssembly. Guards threading, file I/O, networking. # _WIN32 : Compiler-defined for Windows. Guards POSIX APIs (mmap, signals, IPC). -# BB_LITE : CMake option. Excludes server-side subsystems (lmdb, world_state, ipc, nodejs_module). +# BB_LITE : CMake option. Excludes server-side subsystems (ipc, vm2_wsdb, cdb, avm). # Enabled for: iOS, Android, Windows cross-compiles. # WASM : CMake variable mirroring __wasm__ for CMake-level logic. @@ -45,7 +45,7 @@ option(ENABLE_PIC "Builds with position independent code" OFF) option(SYNTAX_ONLY "only check syntax (-fsyntax-only)" OFF) option(ENABLE_WASM_BENCH "Enable BB_BENCH benchmarking support in WASM builds (dev only, not for releases)" OFF) option(AVM "enable building of vm2 module and bb-avm" ON) -option(BB_LITE "Exclude server-side subsystems: lmdb, world_state, ipc, nodejs_module" OFF) +option(BB_LITE "Exclude server-side subsystems: ipc, vm2_wsdb, cdb, avm" OFF) if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") message(STATUS "Compiling for ARM.") diff --git a/barretenberg/cpp/CMakePresets.json b/barretenberg/cpp/CMakePresets.json index ba43f3821a2d..45a9533377b3 100644 --- a/barretenberg/cpp/CMakePresets.json +++ b/barretenberg/cpp/CMakePresets.json @@ -801,25 +801,25 @@ "name": "amd64-linux", "configurePreset": "amd64-linux", "inheritConfigureEnvironment": true, - "targets": ["bb", "nodejs_module", "bb-external", "aztec-wsdb", "bb-avm-sim"] + "targets": ["bb", "nodejs_module", "bb-external", "bb-avm-sim"] }, { "name": "arm64-linux", "configurePreset": "arm64-linux", "inheritConfigureEnvironment": true, - "targets": ["bb", "nodejs_module", "bb-external", "aztec-wsdb", "bb-avm-sim"] + "targets": ["bb", "nodejs_module", "bb-external", "bb-avm-sim"] }, { "name": "amd64-macos", "configurePreset": "amd64-macos", "inheritConfigureEnvironment": true, - "targets": ["bb", "nodejs_module", "bb-external", "aztec-wsdb", "bb-avm-sim"] + "targets": ["bb", "nodejs_module", "bb-external", "bb-avm-sim"] }, { "name": "arm64-macos", "configurePreset": "arm64-macos", "inheritConfigureEnvironment": true, - "targets": ["bb", "nodejs_module", "bb-external", "aztec-wsdb", "bb-avm-sim"] + "targets": ["bb", "nodejs_module", "bb-external", "bb-avm-sim"] }, { "name": "amd64-windows", diff --git a/barretenberg/cpp/scripts/audit/audit_scopes/merkle_tree_audit_scope.md b/barretenberg/cpp/scripts/audit/audit_scopes/merkle_tree_audit_scope.md index 8903716cb126..d1d10edf40a0 100644 --- a/barretenberg/cpp/scripts/audit/audit_scopes/merkle_tree_audit_scope.md +++ b/barretenberg/cpp/scripts/audit/audit_scopes/merkle_tree_audit_scope.md @@ -20,14 +20,14 @@ Note: Paths relative to `aztec-packages/barretenberg/cpp/src/barretenberg` ### Node Store (Cache + Metadata) 8. `crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp` 9. `crypto/merkle_tree/node_store/content_addressed_cache.hpp` -10. `crypto/merkle_tree/node_store/tree_meta.hpp` +10. `crypto/merkle_tree/tree_meta.hpp` ### Append-Only Tree 11. `crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp` ### Indexed Tree 12. `crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp` -13. `crypto/merkle_tree/indexed_tree/indexed_leaf.hpp` +13. `crypto/merkle_tree/indexed_leaf.hpp` ## Summary of Module diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt index 47679fc24fc0..42fd690a956e 100644 --- a/barretenberg/cpp/src/CMakeLists.txt +++ b/barretenberg/cpp/src/CMakeLists.txt @@ -115,22 +115,17 @@ add_subdirectory(barretenberg/ultra_honk) add_subdirectory(barretenberg/vm2_stub) add_subdirectory(barretenberg/wasi) -if(NOT BB_LITE) - add_subdirectory(barretenberg/lmdblib) -endif() - if(NOT FUZZING AND NOT WASM AND NOT BB_LITE) - # Fuzzing preset cannot be built with world_state as world_state cannot compile with MULTITHREADING=OFF - # Mobile builds exclude these modules that require LMDB or aren't needed on mobile + # Mobile builds exclude these modules that aren't needed on mobile add_subdirectory(barretenberg/aztec) - add_subdirectory(barretenberg/world_state) # NOTE: Do not conditionally base this on the AVM flag as it defines a necessary vm2_sim library. add_subdirectory(barretenberg/vm2) add_subdirectory(barretenberg/ipc) - add_subdirectory(barretenberg/wsdb) add_subdirectory(barretenberg/vm2_wsdb) add_subdirectory(barretenberg/cdb) add_subdirectory(barretenberg/avm) + # nodejs_module hosts only the msgpack_client NAPI wrappers now (bb.js's SHM + # transport). The lmdb_store kv-store NAPI moved to native-packages/kvdb. add_subdirectory(barretenberg/nodejs_module) endif() @@ -146,7 +141,6 @@ if(FUZZING_AVM) if(FUZZING) # Only add these if they weren't added above (when NOT FUZZING AND NOT WASM) add_subdirectory(barretenberg/aztec) - add_subdirectory(barretenberg/world_state) add_subdirectory(barretenberg/vm2) endif() add_subdirectory(barretenberg/avm_fuzzer) @@ -215,11 +209,9 @@ set(BARRETENBERG_TARGET_OBJECTS set(BB_EXTERNAL_TARGET_OBJECTS ${BARRETENBERG_TARGET_OBJECTS}) if(NOT WASM AND NOT FUZZING AND NOT BB_LITE) - # enable merkle trees and lmdb (not for mobile builds) + # enable merkle trees (not for mobile builds) list(APPEND BARRETENBERG_TARGET_OBJECTS $) - list(APPEND BARRETENBERG_TARGET_OBJECTS $) list(APPEND BARRETENBERG_TARGET_OBJECTS $) - list(APPEND BARRETENBERG_TARGET_OBJECTS $) endif() add_library( diff --git a/barretenberg/cpp/src/barretenberg/avm/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/avm/CMakeLists.txt index 16610a355d99..76acead0a975 100644 --- a/barretenberg/cpp/src/barretenberg/avm/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/avm/CMakeLists.txt @@ -49,7 +49,6 @@ target_link_libraries( barretenberg vm2_sim wsdb_ipc_merkle_db - wsdb_ipc_client cdb_ipc_client ipc_runtime env diff --git a/barretenberg/cpp/src/barretenberg/avm/avm_execute.cpp b/barretenberg/cpp/src/barretenberg/avm/avm_execute.cpp index 4d00b7ec156a..f10014c7401f 100644 --- a/barretenberg/cpp/src/barretenberg/avm/avm_execute.cpp +++ b/barretenberg/cpp/src/barretenberg/avm/avm_execute.cpp @@ -11,7 +11,7 @@ namespace bb::avm { using namespace bb::avm2; -using namespace bb::world_state; +using bb::crypto::merkle_tree::WorldStateRevision; // Cancellation for the single in-flight simulation. bb-avm-sim runs exactly one // simulation at a time; the SIGUSR1 handler (which may run on any thread) cancels diff --git a/barretenberg/cpp/src/barretenberg/avm/avm_execute.hpp b/barretenberg/cpp/src/barretenberg/avm/avm_execute.hpp index 4f9a18f0b33f..37f2a85dd5c7 100644 --- a/barretenberg/cpp/src/barretenberg/avm/avm_execute.hpp +++ b/barretenberg/cpp/src/barretenberg/avm/avm_execute.hpp @@ -7,7 +7,7 @@ #include "barretenberg/avm/generated/avm_ipc_server.hpp" #include "barretenberg/cdb/cdb_ipc_client.hpp" #include "barretenberg/vm2/simulation/lib/cancellation_token.hpp" -#include "barretenberg/wsdb/generated/wsdb_ipc_client.hpp" +#include "barretenberg/vm2_wsdb/generated/wsdb_ipc_client.hpp" #include diff --git a/barretenberg/cpp/src/barretenberg/avm/avm_ipc_server.cpp b/barretenberg/cpp/src/barretenberg/avm/avm_ipc_server.cpp index 80f3fed08372..4f2443edacdb 100644 --- a/barretenberg/cpp/src/barretenberg/avm/avm_ipc_server.cpp +++ b/barretenberg/cpp/src/barretenberg/avm/avm_ipc_server.cpp @@ -2,7 +2,7 @@ #include "barretenberg/avm/avm_execute.hpp" #include "barretenberg/cdb/cdb_ipc_client.hpp" #include "barretenberg/common/log.hpp" -#include "barretenberg/wsdb/generated/wsdb_ipc_client.hpp" +#include "barretenberg/vm2_wsdb/generated/wsdb_ipc_client.hpp" #include "ipc_runtime/ipc_server.hpp" #include "ipc_runtime/serve_helper.hpp" #include "ipc_runtime/signal_handlers.hpp" diff --git a/barretenberg/cpp/src/barretenberg/avm_fuzzer/common/interfaces/dbs.cpp b/barretenberg/cpp/src/barretenberg/avm_fuzzer/common/interfaces/dbs.cpp index b7587ae6ad90..8ecf61976124 100644 --- a/barretenberg/cpp/src/barretenberg/avm_fuzzer/common/interfaces/dbs.cpp +++ b/barretenberg/cpp/src/barretenberg/avm_fuzzer/common/interfaces/dbs.cpp @@ -6,7 +6,7 @@ #include "barretenberg/avm_fuzzer/fuzz_lib/constants.hpp" #include "barretenberg/aztec/aztec_constants.hpp" #include "barretenberg/common/throw_or_abort.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/crypto/poseidon2/poseidon2.hpp" #include "barretenberg/vm2/common/aztec_types.hpp" #include "barretenberg/vm2/simulation/lib/contract_crypto.hpp" diff --git a/barretenberg/cpp/src/barretenberg/avm_fuzzer/common/interfaces/dbs.hpp b/barretenberg/cpp/src/barretenberg/avm_fuzzer/common/interfaces/dbs.hpp index ae63df381506..69616e538bb4 100644 --- a/barretenberg/cpp/src/barretenberg/avm_fuzzer/common/interfaces/dbs.hpp +++ b/barretenberg/cpp/src/barretenberg/avm_fuzzer/common/interfaces/dbs.hpp @@ -3,7 +3,7 @@ #include #include -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/vm2/common/aztec_types.hpp" #include "barretenberg/vm2/simulation/interfaces/db.hpp" #include "barretenberg/vm2/simulation/lib/memory_merkle_db.hpp" diff --git a/barretenberg/cpp/src/barretenberg/avm_fuzzer/fuzz_lib/simulator.hpp b/barretenberg/cpp/src/barretenberg/avm_fuzzer/fuzz_lib/simulator.hpp index 60f020336b52..e334f794cfba 100644 --- a/barretenberg/cpp/src/barretenberg/avm_fuzzer/fuzz_lib/simulator.hpp +++ b/barretenberg/cpp/src/barretenberg/avm_fuzzer/fuzz_lib/simulator.hpp @@ -4,7 +4,7 @@ #include "barretenberg/avm_fuzzer/common/interfaces/dbs.hpp" #include "barretenberg/avm_fuzzer/common/process.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/vm2/common/avm_io.hpp" #include "barretenberg/vm2/common/aztec_types.hpp" #include "barretenberg/vm2/common/field.hpp" diff --git a/barretenberg/cpp/src/barretenberg/avm_fuzzer/fuzzer_lib.hpp b/barretenberg/cpp/src/barretenberg/avm_fuzzer/fuzzer_lib.hpp index a3c328193ad1..5cb2d402b9c4 100644 --- a/barretenberg/cpp/src/barretenberg/avm_fuzzer/fuzzer_lib.hpp +++ b/barretenberg/cpp/src/barretenberg/avm_fuzzer/fuzzer_lib.hpp @@ -11,7 +11,7 @@ #include "barretenberg/avm_fuzzer/fuzz_lib/simulator.hpp" #include "barretenberg/avm_fuzzer/mutations/bytecode.hpp" #include "barretenberg/avm_fuzzer/mutations/tx_data.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/serialize/msgpack_impl.hpp" #include "barretenberg/vm2/common/avm_io.hpp" #include "barretenberg/vm2/common/aztec_types.hpp" diff --git a/barretenberg/cpp/src/barretenberg/avm_fuzzer/mutations/bytecode.hpp b/barretenberg/cpp/src/barretenberg/avm_fuzzer/mutations/bytecode.hpp index 6cb28e9aa3f9..02f56291017d 100644 --- a/barretenberg/cpp/src/barretenberg/avm_fuzzer/mutations/bytecode.hpp +++ b/barretenberg/cpp/src/barretenberg/avm_fuzzer/mutations/bytecode.hpp @@ -1,6 +1,6 @@ #pragma once -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/vm2/common/aztec_types.hpp" #include #include diff --git a/barretenberg/cpp/src/barretenberg/aztec/aztec_hash_policy.hpp b/barretenberg/cpp/src/barretenberg/aztec/aztec_hash_policy.hpp index 7f172ce1a7e7..5ccc0cca10e2 100644 --- a/barretenberg/cpp/src/barretenberg/aztec/aztec_hash_policy.hpp +++ b/barretenberg/cpp/src/barretenberg/aztec/aztec_hash_policy.hpp @@ -2,7 +2,7 @@ #include "barretenberg/aztec/aztec_constants.hpp" #include "barretenberg/crypto/merkle_tree/hash.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" namespace bb::aztec { diff --git a/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt index 2103fa9f89f8..2916043712d8 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt @@ -8,7 +8,5 @@ add_subdirectory(relations_bench) add_subdirectory(sumcheck_bench) add_subdirectory(translator_bench) add_subdirectory(poseidon2_bench) -add_subdirectory(indexed_tree_bench) -add_subdirectory(append_only_tree_bench) add_subdirectory(ultra_bench) add_subdirectory(circuit_construction_bench) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt deleted file mode 100644 index 7182d4b4be0d..000000000000 --- a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -barretenberg_module(append_only_tree_bench crypto_poseidon2 ext_starknet_crypto_poseidon crypto_pedersen_hash crypto_merkle_tree) diff --git a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt deleted file mode 100644 index 37ac8e31a80c..000000000000 --- a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -barretenberg_module(indexed_tree_bench crypto_poseidon2 ext_starknet_crypto_poseidon crypto_pedersen_hash crypto_merkle_tree) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt index 55272c0b51b2..f7af171f7984 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt @@ -1,7 +1,6 @@ # merkle tree is agnostic to hash function barretenberg_module( crypto_merkle_tree - lmdblib ) # Need -fPIC because it's linked into the nodejs_module shared library set_target_properties(crypto_merkle_tree_objects PROPERTIES POSITION_INDEPENDENT_CODE ON) diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_leaf.hpp similarity index 100% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_leaf.hpp diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp index 420d92f163cf..bf0b5df2b47c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp @@ -7,9 +7,8 @@ #pragma once #include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/tree_meta.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" #include "barretenberg/serialize/msgpack.hpp" @@ -218,42 +217,6 @@ struct GetLowIndexedLeafResponse { } }; -struct CommitResponse { - TreeMeta meta; - TreeDBStats stats; - - CommitResponse() = default; - ~CommitResponse() = default; - CommitResponse(const CommitResponse& other) = default; - CommitResponse(CommitResponse&& other) noexcept = default; - CommitResponse& operator=(const CommitResponse& other) = default; - CommitResponse& operator=(CommitResponse&& other) noexcept = default; -}; - -struct UnwindResponse { - TreeMeta meta; - TreeDBStats stats; - - UnwindResponse() = default; - ~UnwindResponse() = default; - UnwindResponse(const UnwindResponse& other) = default; - UnwindResponse(UnwindResponse&& other) noexcept = default; - UnwindResponse& operator=(const UnwindResponse& other) = default; - UnwindResponse& operator=(UnwindResponse&& other) noexcept = default; -}; - -struct RemoveHistoricResponse { - TreeMeta meta; - TreeDBStats stats; - - RemoveHistoricResponse() = default; - ~RemoveHistoricResponse() = default; - RemoveHistoricResponse(const RemoveHistoricResponse& other) = default; - RemoveHistoricResponse(RemoveHistoricResponse&& other) noexcept = default; - RemoveHistoricResponse& operator=(const RemoveHistoricResponse& other) = default; - RemoveHistoricResponse& operator=(RemoveHistoricResponse&& other) noexcept = default; -}; - template struct TypedResponse { ResponseType inner; bool success{ true }; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/tree_meta.hpp similarity index 100% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp rename to barretenberg/cpp/src/barretenberg/crypto/merkle_tree/tree_meta.hpp diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp index 2af3fb57c206..edf724c4213c 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp +++ b/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp @@ -7,14 +7,10 @@ #pragma once #include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/lmdblib/types.hpp" -#include "lmdb.h" #include #include namespace bb::crypto::merkle_tree { -using namespace bb::lmdblib; - using index_t = uint64_t; using block_number_t = uint32_t; using LeafIndexKeyType = uint64_t; @@ -65,81 +61,4 @@ const std::string LEAF_PREIMAGES_DB = "leaf preimages"; const std::string LEAF_INDICES_DB = "leaf indices"; const std::string BLOCK_INDICES_DB = "block indices"; -struct TreeDBStats { - uint64_t mapSize; - uint64_t physicalFileSize; - DBStats blocksDBStats; - DBStats nodesDBStats; - DBStats leafPreimagesDBStats; - DBStats leafIndicesDBStats; - DBStats blockIndicesDBStats; - - TreeDBStats() = default; - TreeDBStats(uint64_t mapSize, uint64_t physicalFileSize) - : mapSize(mapSize) - , physicalFileSize(physicalFileSize) - {} - TreeDBStats(uint64_t mapSize, - uint64_t physicalFileSize, - const DBStats& blockStats, - const DBStats& nodesStats, - const DBStats& leafPreimagesDBStats, - const DBStats& leafIndicesStats, - const DBStats& blockIndicesStats) - : mapSize(mapSize) - , physicalFileSize(physicalFileSize) - , blocksDBStats(blockStats) - , nodesDBStats(nodesStats) - , leafPreimagesDBStats(leafPreimagesDBStats) - , leafIndicesDBStats(leafIndicesStats) - , blockIndicesDBStats(blockIndicesStats) - {} - TreeDBStats(const TreeDBStats& other) = default; - TreeDBStats(TreeDBStats&& other) noexcept { *this = std::move(other); } - - ~TreeDBStats() = default; - - SERIALIZATION_FIELDS(mapSize, - physicalFileSize, - blocksDBStats, - nodesDBStats, - leafPreimagesDBStats, - leafIndicesDBStats, - blockIndicesDBStats) - - bool operator==(const TreeDBStats& other) const - { - return mapSize == other.mapSize && physicalFileSize == other.physicalFileSize && - blocksDBStats == other.blocksDBStats && nodesDBStats == other.nodesDBStats && - leafPreimagesDBStats == other.leafPreimagesDBStats && leafIndicesDBStats == other.leafIndicesDBStats && - blockIndicesDBStats == other.blockIndicesDBStats; - } - - TreeDBStats& operator=(TreeDBStats&& other) noexcept - { - if (this != &other) { - mapSize = other.mapSize; - physicalFileSize = other.physicalFileSize; - blocksDBStats = std::move(other.blocksDBStats); - nodesDBStats = std::move(other.nodesDBStats); - leafPreimagesDBStats = std::move(other.leafPreimagesDBStats); - leafIndicesDBStats = std::move(other.leafIndicesDBStats); - blockIndicesDBStats = std::move(other.blockIndicesDBStats); - } - return *this; - } - - TreeDBStats& operator=(const TreeDBStats& other) = default; - - friend std::ostream& operator<<(std::ostream& os, const TreeDBStats& stats) - { - os << "Map Size: " << stats.mapSize << ", Physical File Size: " << stats.physicalFileSize << " Blocks DB " - << stats.blocksDBStats << ", Nodes DB " << stats.nodesDBStats << ", Leaf Pre-images DB " - << stats.leafPreimagesDBStats << ", Leaf Indices DB " << stats.leafIndicesDBStats << ", Block Indices DB " - << stats.blockIndicesDBStats; - return os; - } -}; - -std::ostream& operator<<(std::ostream& os, const TreeDBStats& stats); } // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/lmdblib/CMakeLists.txt deleted file mode 100644 index efe372413808..000000000000 --- a/barretenberg/cpp/src/barretenberg/lmdblib/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -barretenberg_module(lmdblib lmdb numeric) -set_target_properties(lmdblib_objects PROPERTIES POSITION_INDEPENDENT_CODE ON) diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/fixtures.hpp b/barretenberg/cpp/src/barretenberg/lmdblib/fixtures.hpp deleted file mode 100644 index 2e6cc670dba4..000000000000 --- a/barretenberg/cpp/src/barretenberg/lmdblib/fixtures.hpp +++ /dev/null @@ -1,40 +0,0 @@ -#include "barretenberg/lmdblib/types.hpp" -#include "barretenberg/numeric/random/engine.hpp" -#include - -namespace bb::lmdblib { -const uint32_t NUM_VALUES = 1024; -inline auto& engine = numeric::get_debug_randomness(); -inline auto& random_engine = numeric::get_randomness(); - -inline std::string random_string() -{ - std::stringstream ss; - ss << random_engine.get_random_uint32(); - return ss.str(); -} - -inline std::string random_temp_directory() -{ - std::stringstream ss; - ss << "/tmp/lmdb/" << random_string(); - return ss.str(); -} - -inline std::vector serialise(std::string key) -{ - std::vector data(key.begin(), key.end()); - return data; -} - -inline Key get_key(int64_t keyCount) -{ - return serialise((std::stringstream() << "Key" << keyCount).str()); -} - -inline Value get_value(int64_t keyCount, int64_t valueCount) -{ - return serialise((std::stringstream() << "Key" << keyCount << "Data" << valueCount).str()); -} - -} // namespace bb::lmdblib \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_helpers.cpp b/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_helpers.cpp deleted file mode 100644 index fa4932a7d649..000000000000 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_helpers.cpp +++ /dev/null @@ -1,83 +0,0 @@ -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/numeric/uint256/uint256.hpp" -#include "lmdb.h" -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#define htole64(x) OSSwapHostToLittleInt64(x) -#define le64toh(x) OSSwapLittleToHostInt64(x) -#else -#include -#endif - -namespace bb::lmdblib { -void throw_error(const std::string& errorString, int error) -{ - std::stringstream ss; - ss << errorString << ": " << error << " - " << mdb_strerror(error) << std::endl; - throw std::runtime_error(ss.str()); -} - -std::vector serialise_key(uint8_t key) -{ - return { key }; -} - -void deserialise_key(void* data, uint8_t& key) -{ - uint8_t* p = static_cast(data); - key = *p; -} - -// 64 bit integers are stored in little endian byte order -std::vector serialise_key(uint64_t key) -{ - uint64_t le = htole64(key); - const uint8_t* p = reinterpret_cast(&le); - return std::vector(p, p + sizeof(key)); -} - -void deserialise_key(void* data, uint64_t& key) -{ - uint64_t le = 0; - std::memcpy(&le, data, sizeof(le)); - key = le64toh(le); -} - -std::vector serialise_key(const uint256_t& key) -{ - std::vector buf(32); - std::memcpy(buf.data(), key.data, 32); - return buf; -} - -void deserialise_key(void* data, uint256_t& key) -{ - std::memcpy(key.data, data, 32); -} - -int size_cmp(const MDB_val* a, const MDB_val* b) -{ - if (a->mv_size < b->mv_size) { - return -1; - } - return (a->mv_size > b->mv_size) ? 1 : 0; -} - -std::vector mdb_val_to_vector(const MDB_val& dbVal) -{ - const uint8_t* p = static_cast(dbVal.mv_data); - return std::vector(p, p + dbVal.mv_size); -} - -void copy_to_vector(const MDB_val& dbVal, std::vector& target) -{ - std::vector temp = mdb_val_to_vector(dbVal); - target.swap(temp); -} -} // namespace bb::lmdblib \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_helpers.hpp b/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_helpers.hpp deleted file mode 100644 index 6117fc94a6f0..000000000000 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_helpers.hpp +++ /dev/null @@ -1,61 +0,0 @@ - -#pragma once -#include "barretenberg/numeric/uint256/uint256.hpp" -#include "lmdb.h" -#include -#include -namespace bb::lmdblib { -void throw_error(const std::string& errorString, int error); - -int size_cmp(const MDB_val* a, const MDB_val* b); - -std::vector serialise_key(uint8_t key); -std::vector serialise_key(uint64_t key); -std::vector serialise_key(const uint256_t& key); - -void deserialise_key(void* data, uint8_t& key); -void deserialise_key(void* data, uint64_t& key); -void deserialise_key(void* data, uint256_t& key); - -template int value_cmp(const MDB_val* a, const MDB_val* b) -{ - T lhs; - T rhs; - deserialise_key(a->mv_data, lhs); - deserialise_key(b->mv_data, rhs); - if (lhs < rhs) { - return -1; - } - if (lhs > rhs) { - return 1; - } - return 0; -} - -std::vector mdb_val_to_vector(const MDB_val& dbVal); -void copy_to_vector(const MDB_val& dbVal, std::vector& target); - -template bool call_lmdb_func(int (*f)(TArgs...), TArgs... args) -{ - int error = f(args...); - return error == 0; -} - -template int call_lmdb_func_with_return(int (*f)(TArgs...), TArgs... args) -{ - return f(args...); -} - -template void call_lmdb_func(const std::string& errorString, int (*f)(TArgs...), TArgs... args) -{ - int error = f(args...); - if (error != 0) { - throw_error(errorString, error); - } -} - -template void call_lmdb_func(void (*f)(TArgs...), TArgs... args) -{ - f(args...); -} -} // namespace bb::lmdblib \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/types.hpp b/barretenberg/cpp/src/barretenberg/lmdblib/types.hpp deleted file mode 100644 index 68b48091f2de..000000000000 --- a/barretenberg/cpp/src/barretenberg/lmdblib/types.hpp +++ /dev/null @@ -1,69 +0,0 @@ -#pragma once - -#include "barretenberg/serialize/msgpack.hpp" -#include "lmdb.h" -#include -#include -#include -#include -#include -namespace bb::lmdblib { -using Key = std::vector; -using Value = std::vector; -using KeysVector = std::vector; -using ValuesVector = std::vector; -using KeyValuesPair = std::pair; -using OptionalValues = std::optional; -using OptionalValuesVector = std::vector; -using KeyDupValuesVector = std::vector; -using KeyOptionalValuesPair = std::pair; -using KeyOptionalValuesVector = std::vector; - -struct DBStats { - std::string name; - uint64_t numDataItems; - uint64_t totalUsedSize; - - DBStats() = default; - DBStats(const DBStats& other) = default; - DBStats(DBStats&& other) noexcept { *this = std::move(other); } - ~DBStats() = default; - DBStats(std::string name, MDB_stat& stat) - : name(std::move(name)) - , numDataItems(stat.ms_entries) - , totalUsedSize(stat.ms_psize * (stat.ms_branch_pages + stat.ms_leaf_pages + stat.ms_overflow_pages)) - {} - DBStats(const std::string& name, uint64_t numDataItems, uint64_t totalUsedSize) - : name(name) - , numDataItems(numDataItems) - , totalUsedSize(totalUsedSize) - {} - - SERIALIZATION_FIELDS(name, numDataItems, totalUsedSize) - - bool operator==(const DBStats& other) const - { - return name == other.name && numDataItems == other.numDataItems && totalUsedSize == other.totalUsedSize; - } - - DBStats& operator=(const DBStats& other) = default; - - DBStats& operator=(DBStats&& other) noexcept - { - if (this != &other) { - name = std::move(other.name); - numDataItems = other.numDataItems; - totalUsedSize = other.totalUsedSize; - } - return *this; - } - - friend std::ostream& operator<<(std::ostream& os, const DBStats& stats) - { - os << "DB " << stats.name << ", num items: " << stats.numDataItems - << ", total used size: " << stats.totalUsedSize; - return os; - } -}; - -} // namespace bb::lmdblib \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/nodejs_module/CMakeLists.txt index 736f67b83f2b..77d07c8119eb 100644 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/nodejs_module/CMakeLists.txt @@ -27,7 +27,7 @@ string(REGEX REPLACE "[\r\n\"]" "" NODE_API_HEADERS_DIR ${NODE_API_HEADERS_DIR}) add_library(nodejs_module SHARED ${SOURCE_FILES}) set_target_properties(nodejs_module PROPERTIES PREFIX "" SUFFIX ".node") target_include_directories(nodejs_module PRIVATE ${NODE_API_HEADERS_DIR} ${NODE_ADDON_API_DIR}) -target_link_libraries(nodejs_module PRIVATE ipc ipc_runtime lmdblib) +target_link_libraries(nodejs_module PRIVATE ipc ipc_runtime) # On macOS, Node.js N-API symbols are provided by the runtime, not at link time if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/init_module.cpp b/barretenberg/cpp/src/barretenberg/nodejs_module/init_module.cpp index 1a5a0d0dd396..5dcc6eade9a6 100644 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/init_module.cpp +++ b/barretenberg/cpp/src/barretenberg/nodejs_module/init_module.cpp @@ -1,11 +1,9 @@ -#include "barretenberg/nodejs_module/lmdb_store/lmdb_store_wrapper.hpp" #include "barretenberg/nodejs_module/msgpack_client/msgpack_client_async.hpp" #include "barretenberg/nodejs_module/msgpack_client/msgpack_client_wrapper.hpp" #include "napi.h" Napi::Object Init(Napi::Env env, Napi::Object exports) { - exports.Set(Napi::String::New(env, "LMDBStore"), bb::nodejs::lmdb_store::LMDBStoreWrapper::get_class(env)); exports.Set(Napi::String::New(env, "MsgpackClient"), bb::nodejs::msgpack_client::MsgpackClientWrapper::get_class(env)); exports.Set(Napi::String::New(env, "MsgpackClientAsync"), diff --git a/barretenberg/cpp/src/barretenberg/vm2/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/vm2/CMakeLists.txt index 5fb004640f6c..8f1b2123e950 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/vm2/CMakeLists.txt @@ -53,10 +53,8 @@ if(AVM) # vm2_tests needs dsl for integration testing, but this creates circular dependency: # vm2 provides functions that dsl calls, but tests need dsl functions. # Solution: Link dsl, then vm2 again to resolve the circular reference. - # world_state is a test-only dependency: memory_merkle_db.test.cpp checks MemoryMerkleDB against a - # real WorldState, but vm2 itself no longer depends on world_state. if(NOT WASM AND NOT FUZZING) - target_link_libraries(vm2_tests PRIVATE dsl vm2 world_state) + target_link_libraries(vm2_tests PRIVATE dsl vm2) endif() if(FUZZING) diff --git a/barretenberg/cpp/src/barretenberg/vm2/common/avm_io.cpp b/barretenberg/cpp/src/barretenberg/vm2/common/avm_io.cpp index 3338bae70dab..0dfbe96728fc 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/common/avm_io.cpp +++ b/barretenberg/cpp/src/barretenberg/vm2/common/avm_io.cpp @@ -4,6 +4,7 @@ #include "barretenberg/aztec/aztec_constants.hpp" #include "barretenberg/serialize/msgpack.hpp" +#include "barretenberg/serialize/msgpack_impl.hpp" namespace bb::avm2 { namespace { diff --git a/barretenberg/cpp/src/barretenberg/vm2/common/avm_io.hpp b/barretenberg/cpp/src/barretenberg/vm2/common/avm_io.hpp index ef5bfcaf5153..2d18d714ab73 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/common/avm_io.hpp +++ b/barretenberg/cpp/src/barretenberg/vm2/common/avm_io.hpp @@ -6,7 +6,7 @@ #include "barretenberg/common/streams.hpp" // Derives operator<< from SERIALIZATION_FIELDS. #include "barretenberg/common/utils.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/merkle_tree_id.hpp" // For MerkleTreeId, WorldStateRevision and MSGPACK_ADD_ENUM(MerkleTreeId). #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/serialize/msgpack.hpp" diff --git a/barretenberg/cpp/src/barretenberg/vm2/simulation/gadgets/update_check.test.cpp b/barretenberg/cpp/src/barretenberg/vm2/simulation/gadgets/update_check.test.cpp index 9a8693fd9f2b..088c7b1a56dd 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/simulation/gadgets/update_check.test.cpp +++ b/barretenberg/cpp/src/barretenberg/vm2/simulation/gadgets/update_check.test.cpp @@ -212,8 +212,8 @@ TEST_P(UpdateCheckHashNonzeroTest, WithHash) EXPECT_CALL(merkle_db, get_tree_state()).WillRepeatedly(Return(tree_states)); EXPECT_CALL(merkle_db, as_unconstrained()).WillRepeatedly(ReturnRef(mock_low_level_merkle_db)); - EXPECT_CALL(mock_low_level_merkle_db, get_low_indexed_leaf(world_state::MerkleTreeId::PUBLIC_DATA_TREE, _)) - .WillRepeatedly([&](world_state::MerkleTreeId, const FF& leaf_slot) { + EXPECT_CALL(mock_low_level_merkle_db, get_low_indexed_leaf(crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, _)) + .WillRepeatedly([&](crypto::merkle_tree::MerkleTreeId, const FF& leaf_slot) { for (size_t i = 0; i < update_preimage_slots.size(); ++i) { if (leaf_slot == update_preimage_slots[i]) { return GetLowIndexedLeafResponse(true, static_cast(i)); @@ -297,8 +297,8 @@ TEST(AvmSimulationUpdateCheck, HashMismatch) EXPECT_CALL(mock_low_level_merkle_db, get_tree_roots()).WillRepeatedly(Return(trees)); EXPECT_CALL(merkle_db, as_unconstrained()).WillRepeatedly(ReturnRef(mock_low_level_merkle_db)); - EXPECT_CALL(mock_low_level_merkle_db, get_low_indexed_leaf(world_state::MerkleTreeId::PUBLIC_DATA_TREE, _)) - .WillRepeatedly([&](world_state::MerkleTreeId, const FF& leaf_slot) { + EXPECT_CALL(mock_low_level_merkle_db, get_low_indexed_leaf(crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, _)) + .WillRepeatedly([&](crypto::merkle_tree::MerkleTreeId, const FF& leaf_slot) { return GetLowIndexedLeafResponse(true, static_cast(leaf_slot - delayed_public_mutable_leaf_slot)); }); diff --git a/barretenberg/cpp/src/barretenberg/vm2/simulation/interfaces/db.hpp b/barretenberg/cpp/src/barretenberg/vm2/simulation/interfaces/db.hpp index 284dc69173d7..7fadf284ec19 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/simulation/interfaces/db.hpp +++ b/barretenberg/cpp/src/barretenberg/vm2/simulation/interfaces/db.hpp @@ -4,7 +4,7 @@ #include #include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/merkle_tree_id.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" diff --git a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/db_types.hpp b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/db_types.hpp index 1d5e12091595..4de9e949be8d 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/db_types.hpp +++ b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/db_types.hpp @@ -1,6 +1,6 @@ #pragma once -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/merkle_tree_id.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/vm2/common/aztec_types.hpp" diff --git a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/hinting_dbs.hpp b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/hinting_dbs.hpp index 5e7cb7d79a5d..a51942c16a07 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/hinting_dbs.hpp +++ b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/hinting_dbs.hpp @@ -5,7 +5,7 @@ #include #include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/vm2/common/avm_io.hpp" #include "barretenberg/vm2/common/aztec_types.hpp" diff --git a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/hinting_dbs.test.cpp b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/hinting_dbs.test.cpp index 1028480441cb..b4e31f5a67b5 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/hinting_dbs.test.cpp +++ b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/hinting_dbs.test.cpp @@ -213,8 +213,8 @@ TEST_F(MockedHintingDBsTest, GetLowLeaf) std::vector update_preimage_slots = { 1, 2, 4 }; // get_low_indexed_leaf will call get_tree_roots: EXPECT_CALL(base_merkle_db, get_tree_roots).Times(static_cast(update_preimage_slots.size())); - EXPECT_CALL(base_merkle_db, get_low_indexed_leaf(world_state::MerkleTreeId::PUBLIC_DATA_TREE, testing::_)) - .WillRepeatedly([&](world_state::MerkleTreeId, const FF& leaf_slot) { + EXPECT_CALL(base_merkle_db, get_low_indexed_leaf(crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, testing::_)) + .WillRepeatedly([&](crypto::merkle_tree::MerkleTreeId, const FF& leaf_slot) { for (size_t i = 0; i < update_preimage_slots.size(); ++i) { if (leaf_slot == update_preimage_slots[i]) { return GetLowIndexedLeafResponse(true, static_cast(i)); @@ -225,30 +225,31 @@ TEST_F(MockedHintingDBsTest, GetLowLeaf) // Call the db: for (const auto& update_preimage_slot : update_preimage_slots) { - hinting_merkle_db.get_low_indexed_leaf(world_state::MerkleTreeId::PUBLIC_DATA_TREE, update_preimage_slot); + hinting_merkle_db.get_low_indexed_leaf(crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, + update_preimage_slot); } ExecutionHints collected_hints; hinting_merkle_db.dump_hints(collected_hints); // Check the collected hints: EXPECT_EQ(collected_hints.get_previous_value_index_hints.size(), update_preimage_slots.size()); - EXPECT_THAT( - collected_hints.get_previous_value_index_hints, - testing::ElementsAreArray({ GetPreviousValueIndexHint{ .hint_key = mock_tree_info.public_data_tree, - .tree_id = world_state::MerkleTreeId::PUBLIC_DATA_TREE, - .value = update_preimage_slots[0], - .index = 0, - .already_present = true }, - GetPreviousValueIndexHint{ .hint_key = mock_tree_info.public_data_tree, - .tree_id = world_state::MerkleTreeId::PUBLIC_DATA_TREE, - .value = update_preimage_slots[1], - .index = 1, - .already_present = true }, - GetPreviousValueIndexHint{ .hint_key = mock_tree_info.public_data_tree, - .tree_id = world_state::MerkleTreeId::PUBLIC_DATA_TREE, - .value = update_preimage_slots[2], - .index = 2, - .already_present = true } })); + EXPECT_THAT(collected_hints.get_previous_value_index_hints, + testing::ElementsAreArray( + { GetPreviousValueIndexHint{ .hint_key = mock_tree_info.public_data_tree, + .tree_id = crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, + .value = update_preimage_slots[0], + .index = 0, + .already_present = true }, + GetPreviousValueIndexHint{ .hint_key = mock_tree_info.public_data_tree, + .tree_id = crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, + .value = update_preimage_slots[1], + .index = 1, + .already_present = true }, + GetPreviousValueIndexHint{ .hint_key = mock_tree_info.public_data_tree, + .tree_id = crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, + .value = update_preimage_slots[2], + .index = 2, + .already_present = true } })); } TEST_F(MockedHintingDBsTest, GetLeafValue) @@ -257,8 +258,8 @@ TEST_F(MockedHintingDBsTest, GetLeafValue) std::vector note_hash_leaf_values = { 11, 22, 44, 88 }; // get_leaf_value will call get_tree_roots: EXPECT_CALL(base_merkle_db, get_tree_roots).Times(static_cast(note_hash_leaf_values.size())); - EXPECT_CALL(base_merkle_db, get_leaf_value(world_state::MerkleTreeId::NOTE_HASH_TREE, testing::_)) - .WillRepeatedly([&](world_state::MerkleTreeId, index_t index) { + EXPECT_CALL(base_merkle_db, get_leaf_value(crypto::merkle_tree::MerkleTreeId::NOTE_HASH_TREE, testing::_)) + .WillRepeatedly([&](crypto::merkle_tree::MerkleTreeId, index_t index) { if (index < note_hash_leaf_values.size()) { return note_hash_leaf_values[index]; } @@ -267,7 +268,7 @@ TEST_F(MockedHintingDBsTest, GetLeafValue) // Call the db: for (index_t i = 0; i < note_hash_leaf_values.size(); i++) { - hinting_merkle_db.get_leaf_value(world_state::MerkleTreeId::NOTE_HASH_TREE, i); + hinting_merkle_db.get_leaf_value(crypto::merkle_tree::MerkleTreeId::NOTE_HASH_TREE, i); } ExecutionHints collected_hints; hinting_merkle_db.dump_hints(collected_hints); @@ -277,19 +278,19 @@ TEST_F(MockedHintingDBsTest, GetLeafValue) EXPECT_THAT(collected_hints.get_leaf_value_hints, testing::ElementsAreArray({ GetLeafValueHint{ .hint_key = mock_tree_info.note_hash_tree, - .tree_id = world_state::MerkleTreeId::NOTE_HASH_TREE, + .tree_id = crypto::merkle_tree::MerkleTreeId::NOTE_HASH_TREE, .index = 0, .value = note_hash_leaf_values[0] }, GetLeafValueHint{ .hint_key = mock_tree_info.note_hash_tree, - .tree_id = world_state::MerkleTreeId::NOTE_HASH_TREE, + .tree_id = crypto::merkle_tree::MerkleTreeId::NOTE_HASH_TREE, .index = 1, .value = note_hash_leaf_values[1] }, GetLeafValueHint{ .hint_key = mock_tree_info.note_hash_tree, - .tree_id = world_state::MerkleTreeId::NOTE_HASH_TREE, + .tree_id = crypto::merkle_tree::MerkleTreeId::NOTE_HASH_TREE, .index = 2, .value = note_hash_leaf_values[2] }, GetLeafValueHint{ .hint_key = mock_tree_info.note_hash_tree, - .tree_id = world_state::MerkleTreeId::NOTE_HASH_TREE, + .tree_id = crypto::merkle_tree::MerkleTreeId::NOTE_HASH_TREE, .index = 3, .value = note_hash_leaf_values[3] }, })); @@ -403,7 +404,7 @@ TEST_F(MockedHintingDBsTest, InsertIndexedLeavesPublicDataTree) EXPECT_THAT(collected_hints.sequential_insert_hints_public_data_tree, testing::ElementsAre(SequentialInsertHint{ .hint_key = state_before, - .tree_id = world_state::MerkleTreeId::PUBLIC_DATA_TREE, + .tree_id = crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, .leaf = public_leaf_value, .low_leaves_witness_data = mock_low_witness_data, .insertion_witness_data = { { public_leaf_value, 1, 6 }, 1, mock_path }, @@ -443,7 +444,7 @@ TEST_F(MockedHintingDBsTest, InsertIndexedLeavesNullifierTree) EXPECT_THAT(collected_hints.sequential_insert_hints_nullifier_tree, testing::ElementsAre(SequentialInsertHint{ .hint_key = state_before, - .tree_id = world_state::MerkleTreeId::NULLIFIER_TREE, + .tree_id = crypto::merkle_tree::MerkleTreeId::NULLIFIER_TREE, .leaf = nullifier, .low_leaves_witness_data = mock_low_witness_data, .insertion_witness_data = { { nullifier, 1, 6 }, 1, mock_path }, @@ -464,10 +465,10 @@ TEST_F(MockedHintingDBsTest, AppendLeaves) EXPECT_CALL(base_merkle_db, get_tree_roots) .WillOnce(testing::Return(mock_tree_info)) .WillOnce(testing::Return(expected_tree_info_after)); - EXPECT_CALL(base_merkle_db, append_leaves(world_state::MerkleTreeId::NOTE_HASH_TREE, testing::_)).Times(1); + EXPECT_CALL(base_merkle_db, append_leaves(crypto::merkle_tree::MerkleTreeId::NOTE_HASH_TREE, testing::_)).Times(1); // Call the db: - hinting_merkle_db.append_leaves(world_state::MerkleTreeId::NOTE_HASH_TREE, note_hash_leaf_values); + hinting_merkle_db.append_leaves(crypto::merkle_tree::MerkleTreeId::NOTE_HASH_TREE, note_hash_leaf_values); ExecutionHints collected_hints; hinting_merkle_db.dump_hints(collected_hints); @@ -476,7 +477,7 @@ TEST_F(MockedHintingDBsTest, AppendLeaves) EXPECT_THAT(collected_hints.append_leaves_hints, testing::ElementsAre(AppendLeavesHint{ .hint_key = initial_state, .state_after = expected_end_state, - .tree_id = world_state::MerkleTreeId::NOTE_HASH_TREE, + .tree_id = crypto::merkle_tree::MerkleTreeId::NOTE_HASH_TREE, .leaves = note_hash_leaf_values })); } diff --git a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/memory_merkle_db.hpp b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/memory_merkle_db.hpp index d1a930692ee2..07d2942cd600 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/memory_merkle_db.hpp +++ b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/memory_merkle_db.hpp @@ -8,7 +8,7 @@ #include "barretenberg/aztec/aztec_hash_policy.hpp" #include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/vm2/common/aztec_types.hpp" #include "barretenberg/vm2/common/field.hpp" diff --git a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/raw_data_dbs.cpp b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/raw_data_dbs.cpp index 96bbeaaa3de1..b2b84b4d1922 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/raw_data_dbs.cpp +++ b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/raw_data_dbs.cpp @@ -9,7 +9,7 @@ #include "barretenberg/aztec/aztec_constants.hpp" #include "barretenberg/common/bb_bench.hpp" #include "barretenberg/common/log.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/vm2/common/avm_io.hpp" #include "barretenberg/vm2/simulation/interfaces/db.hpp" #include "barretenberg/vm2/simulation/lib/contract_crypto.hpp" diff --git a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/raw_data_dbs.hpp b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/raw_data_dbs.hpp index 96ef3a15a1c2..3134bb78dff1 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/raw_data_dbs.hpp +++ b/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/raw_data_dbs.hpp @@ -5,7 +5,7 @@ #include #include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/vm2/common/avm_io.hpp" #include "barretenberg/vm2/common/aztec_types.hpp" diff --git a/barretenberg/cpp/src/barretenberg/vm2/tracegen/update_check_trace.test.cpp b/barretenberg/cpp/src/barretenberg/vm2/tracegen/update_check_trace.test.cpp index 5d0e96d62610..7a0d7226df00 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/tracegen/update_check_trace.test.cpp +++ b/barretenberg/cpp/src/barretenberg/vm2/tracegen/update_check_trace.test.cpp @@ -135,13 +135,13 @@ TEST(UpdateCheckTracegenTest, HashZeroInteractions) uint32_t leaf_index = 27; EXPECT_CALL(mock_low_level_merkle_db, get_tree_roots()).WillRepeatedly(Return(trees)); - EXPECT_CALL(mock_low_level_merkle_db, get_sibling_path(world_state::MerkleTreeId::PUBLIC_DATA_TREE, _)) + EXPECT_CALL(mock_low_level_merkle_db, get_sibling_path(crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, _)) .WillOnce(Return(fr_sibling_path{ 0 })); EXPECT_CALL(mock_low_level_merkle_db, get_leaf_preimage_public_data_tree(_)) .WillOnce(Return(PublicDataTreeLeafPreimage(PublicDataLeafValue(1, 0), 0, 0))); - EXPECT_CALL( - mock_low_level_merkle_db, - get_low_indexed_leaf(world_state::MerkleTreeId::PUBLIC_DATA_TREE, delayed_public_mutable_hash_leaf_slot)) + EXPECT_CALL(mock_low_level_merkle_db, + get_low_indexed_leaf(crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, + delayed_public_mutable_hash_leaf_slot)) .WillOnce(Return(GetLowIndexedLeafResponse(false, leaf_index))); EXPECT_CALL(mock_field_gt, ff_gt(_, _)).WillRepeatedly(Return(true)); @@ -239,7 +239,7 @@ TEST(UpdateCheckTracegenTest, HashNonzeroInteractions) } EXPECT_CALL(mock_low_level_merkle_db, get_tree_roots()).WillRepeatedly(Return(trees)); - EXPECT_CALL(mock_low_level_merkle_db, get_sibling_path(world_state::MerkleTreeId::PUBLIC_DATA_TREE, _)) + EXPECT_CALL(mock_low_level_merkle_db, get_sibling_path(crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, _)) .WillOnce(Return(fr_sibling_path{ 0 })); EXPECT_CALL(mock_low_level_merkle_db, get_leaf_preimage_public_data_tree(_)) .WillRepeatedly([&](const uint64_t& index) { @@ -247,8 +247,8 @@ TEST(UpdateCheckTracegenTest, HashNonzeroInteractions) PublicDataLeafValue(update_leaf_slots[index], update_leaf_values[index]), 0, 0); }); - EXPECT_CALL(mock_low_level_merkle_db, get_low_indexed_leaf(world_state::MerkleTreeId::PUBLIC_DATA_TREE, _)) - .WillRepeatedly([&](world_state::MerkleTreeId, const FF& leaf_slot) { + EXPECT_CALL(mock_low_level_merkle_db, get_low_indexed_leaf(crypto::merkle_tree::MerkleTreeId::PUBLIC_DATA_TREE, _)) + .WillRepeatedly([&](crypto::merkle_tree::MerkleTreeId, const FF& leaf_slot) { for (size_t i = 0; i < update_leaf_slots.size(); ++i) { if (leaf_slot == update_leaf_slots[i]) { return GetLowIndexedLeafResponse(true, static_cast(i)); diff --git a/barretenberg/cpp/src/barretenberg/vm2_wsdb/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/vm2_wsdb/CMakeLists.txt index ee67a00907e3..5c639a33791a 100644 --- a/barretenberg/cpp/src/barretenberg/vm2_wsdb/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/vm2_wsdb/CMakeLists.txt @@ -1,16 +1,61 @@ -# VM2 adapter for native AVM simulation against aztec-wsdb. -if(TARGET vm2_sim AND TARGET wsdb_ipc_client) +# VM2 binding to the wsdb service: the generated IPC client plus the AVM adapter +# (WsdbIpcMerkleDB) that drives native AVM simulation against aztec-wsdb. +# +# The wsdb wire contract (wsdb_schema.jsonc) and the server live in the +# top-level native-packages/wsdb package, which owns the service. bb only +# needs the generated CLIENT, so it codegens from that external schema path. +# This is a deliberate (and currently the only) build-order inversion: +# bb reads native-packages/wsdb/wsdb_schema.jsonc at configure/build time. +if(TARGET vm2_sim) + set(WSDB_SCHEMA ${CMAKE_SOURCE_DIR}/../../native-packages/wsdb/wsdb_schema.jsonc) + set(WSDB_GEN_DIR ${CMAKE_CURRENT_SOURCE_DIR}/generated) + set(WSDB_GEN_OUTPUTS + ${WSDB_GEN_DIR}/wsdb_ipc_client.cpp + ${WSDB_GEN_DIR}/wsdb_ipc_client.hpp + ${WSDB_GEN_DIR}/wsdb_types.hpp + ${WSDB_GEN_DIR}/ipc_codegen/msgpack_adaptor.hpp + ${WSDB_GEN_DIR}/ipc_codegen/named_union.hpp + ${WSDB_GEN_DIR}/ipc_codegen/throw.hpp + ) + set(IPC_CODEGEN_DIR ${CMAKE_SOURCE_DIR}/../../ipc-codegen) + file(GLOB_RECURSE IPC_CODEGEN_SRC + ${IPC_CODEGEN_DIR}/src/*.ts + ${IPC_CODEGEN_DIR}/templates/cpp/*.hpp + ) + add_custom_command( + OUTPUT ${WSDB_GEN_OUTPUTS} + COMMAND node --experimental-strip-types --experimental-transform-types --no-warnings + ${IPC_CODEGEN_DIR}/src/generate.ts + --schema ${WSDB_SCHEMA} + --lang cpp + --out ${WSDB_GEN_DIR} + --client + --cpp-namespace bb::wsdb + --cpp-include-dir barretenberg/vm2_wsdb/generated + DEPENDS ${WSDB_SCHEMA} ${IPC_CODEGEN_SRC} + COMMENT "Generating WSDB IPC client from native-packages/wsdb/wsdb_schema.jsonc" + VERBATIM + ) + add_custom_target(wsdb_ipc_generated DEPENDS ${WSDB_GEN_OUTPUTS}) + add_library( wsdb_ipc_merkle_db STATIC wsdb_ipc_merkle_db.cpp + ${WSDB_GEN_DIR}/wsdb_ipc_client.cpp + ) + add_dependencies(wsdb_ipc_merkle_db wsdb_ipc_generated) + target_include_directories( + wsdb_ipc_merkle_db + PUBLIC + ${WSDB_GEN_DIR} ) target_link_libraries( wsdb_ipc_merkle_db PUBLIC barretenberg vm2_sim - wsdb_ipc_client + ipc_runtime ) set_target_properties(wsdb_ipc_merkle_db PROPERTIES POSITION_INDEPENDENT_CODE ON) endif() diff --git a/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_client.hpp b/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_client.hpp new file mode 100644 index 000000000000..38995831838e --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_client.hpp @@ -0,0 +1,3 @@ +#pragma once + +#include "barretenberg/vm2_wsdb/generated/wsdb_ipc_client.hpp" diff --git a/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.cpp b/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.cpp index 912a840ca71e..1567a9f8ff32 100644 --- a/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.cpp +++ b/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.cpp @@ -1,7 +1,7 @@ #include "barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.hpp" #include "barretenberg/aztec/aztec_constants.hpp" #include "barretenberg/common/log.hpp" -#include "barretenberg/wsdb/wsdb_wire_convert.hpp" +#include "barretenberg/vm2_wsdb/wsdb_wire_convert_client.hpp" #include @@ -26,7 +26,7 @@ using bb::wsdb::tree_id_to_wire; // Constructor // --------------------------------------------------------------------------- -WsdbIpcMerkleDB::WsdbIpcMerkleDB(wsdb::WsdbIpcClient& client, world_state::WorldStateRevision revision) +WsdbIpcMerkleDB::WsdbIpcMerkleDB(wsdb::WsdbIpcClient& client, crypto::merkle_tree::WorldStateRevision revision) : client_(client) , revision_(revision) {} diff --git a/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.hpp b/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.hpp index ff2056b8b6ab..c5b8fcb91471 100644 --- a/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.hpp +++ b/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.hpp @@ -7,9 +7,9 @@ * each LowLevelMerkleDBInterface call into the corresponding WSDB IPC command. */ +#include "barretenberg/crypto/merkle_tree/merkle_tree_id.hpp" #include "barretenberg/vm2/simulation/interfaces/db.hpp" -#include "barretenberg/world_state/types.hpp" -#include "barretenberg/wsdb/wsdb_ipc_client.hpp" +#include "barretenberg/vm2_wsdb/wsdb_ipc_client.hpp" #include #include @@ -23,7 +23,7 @@ class WsdbIpcMerkleDB final : public avm2::simulation::LowLevelMerkleDBInterface * @param client Reference to a connected WsdbIpcClient. * @param revision The world state revision (includes forkId) to use for queries. */ - WsdbIpcMerkleDB(wsdb::WsdbIpcClient& client, world_state::WorldStateRevision revision); + WsdbIpcMerkleDB(wsdb::WsdbIpcClient& client, crypto::merkle_tree::WorldStateRevision revision); avm2::TreeSnapshots get_tree_roots() const override; @@ -58,7 +58,7 @@ class WsdbIpcMerkleDB final : public avm2::simulation::LowLevelMerkleDBInterface void invalidate_tree_roots_cache(); wsdb::WsdbIpcClient& client_; - world_state::WorldStateRevision revision_; + crypto::merkle_tree::WorldStateRevision revision_; std::stack checkpoint_stack_{ { 0 } }; /** Cached tree roots — avoids 5 IPC round trips per get_tree_roots() call. */ mutable std::optional cached_tree_roots_; diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_wire_convert.hpp b/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_wire_convert_client.hpp similarity index 59% rename from barretenberg/cpp/src/barretenberg/wsdb/wsdb_wire_convert.hpp rename to barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_wire_convert_client.hpp index 6de14909e38e..e4e736607afe 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_wire_convert.hpp +++ b/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_wire_convert_client.hpp @@ -1,16 +1,21 @@ #pragma once /** * @file wsdb_wire_convert.hpp - * @brief Wire <-> domain conversion helpers for the aztec-wsdb service. + * @brief Client-side wire <-> domain conversion helpers for the aztec-wsdb service. + * + * These converters depend only on the generic merkle-tree vocabulary + * (crypto/merkle_tree) plus the generated wsdb wire types, so they can be + * compiled into bb's wsdb client (consumed by the AVM simulator) without + * pulling in world_state or the persistent merkle storage. The server-side + * converters that touch world_state aggregates (state references, DB stats, + * tree/world-state meta) live alongside the server in native-packages/wsdb. */ -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/merkle_tree_id.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/lmdblib/types.hpp" -#include "barretenberg/world_state/types.hpp" -#include "barretenberg/world_state/world_state.hpp" -#include "barretenberg/wsdb/generated/wsdb_types.hpp" +#include "barretenberg/vm2_wsdb/generated/wsdb_types.hpp" namespace bb::wsdb { @@ -94,7 +99,7 @@ inline std::vector fr_vec_from_wire(const std::vector& w) return r; } -inline wire::WorldStateRevision revision_to_wire(const world_state::WorldStateRevision& d) +inline wire::WorldStateRevision revision_to_wire(const crypto::merkle_tree::WorldStateRevision& d) { return wire::WorldStateRevision{ .forkId = d.forkId, @@ -103,23 +108,23 @@ inline wire::WorldStateRevision revision_to_wire(const world_state::WorldStateRe }; } -inline world_state::WorldStateRevision revision_from_wire(const wire::WorldStateRevision& w) +inline crypto::merkle_tree::WorldStateRevision revision_from_wire(const wire::WorldStateRevision& w) { - return world_state::WorldStateRevision{ + return crypto::merkle_tree::WorldStateRevision{ .forkId = w.forkId, .blockNumber = w.blockNumber, .includeUncommitted = w.includeUncommitted, }; } -inline MerkleTreeId tree_id_to_wire(world_state::MerkleTreeId d) +inline MerkleTreeId tree_id_to_wire(crypto::merkle_tree::MerkleTreeId d) { return static_cast(d); } -inline world_state::MerkleTreeId tree_id_from_wire(MerkleTreeId w) +inline crypto::merkle_tree::MerkleTreeId tree_id_from_wire(MerkleTreeId w) { - return static_cast(w); + return static_cast(w); } inline wire::PublicDataLeafValue public_data_leaf_to_wire(const crypto::merkle_tree::PublicDataLeafValue& d) @@ -257,7 +262,7 @@ inline std::vector vec_from_wire(const std::vector& w, Fn fn) } inline wire::BatchInsertionResultPublicData batch_public_data_to_wire( - const world_state::BatchInsertionResult& d) + const crypto::merkle_tree::BatchInsertionResult& d) { std::vector sorted; sorted.reserve(d.sorted_leaves.size()); @@ -270,10 +275,10 @@ inline wire::BatchInsertionResultPublicData batch_public_data_to_wire( .subtreePath = fr_vec_to_wire(d.subtree_path) }; } -inline world_state::BatchInsertionResult batch_public_data_from_wire( +inline crypto::merkle_tree::BatchInsertionResult batch_public_data_from_wire( const wire::BatchInsertionResultPublicData& w) { - world_state::BatchInsertionResult r; + crypto::merkle_tree::BatchInsertionResult r; r.low_leaf_witness_data = vec_from_wire>( w.lowLeafWitnessData, public_data_witness_from_wire); @@ -286,7 +291,7 @@ inline world_state::BatchInsertionResult& d) + const crypto::merkle_tree::BatchInsertionResult& d) { std::vector sorted; sorted.reserve(d.sorted_leaves.size()); @@ -299,10 +304,10 @@ inline wire::BatchInsertionResultNullifier batch_nullifier_to_wire( .subtreePath = fr_vec_to_wire(d.subtree_path) }; } -inline world_state::BatchInsertionResult batch_nullifier_from_wire( +inline crypto::merkle_tree::BatchInsertionResult batch_nullifier_from_wire( const wire::BatchInsertionResultNullifier& w) { - world_state::BatchInsertionResult r; + crypto::merkle_tree::BatchInsertionResult r; r.low_leaf_witness_data = vec_from_wire>( w.lowLeafWitnessData, nullifier_witness_from_wire); @@ -315,7 +320,7 @@ inline world_state::BatchInsertionResult& d) + const crypto::merkle_tree::SequentialInsertionResult& d) { return { .lowLeafWitnessData = vec_to_wire(d.low_leaf_witness_data, public_data_witness_to_wire), @@ -323,7 +328,7 @@ inline wire::SequentialInsertionResultPublicData sequential_public_data_to_wire( public_data_witness_to_wire) }; } -inline world_state::SequentialInsertionResult +inline crypto::merkle_tree::SequentialInsertionResult sequential_public_data_from_wire(const wire::SequentialInsertionResultPublicData& w) { return { .low_leaf_witness_data = @@ -335,7 +340,7 @@ sequential_public_data_from_wire(const wire::SequentialInsertionResultPublicData } inline wire::SequentialInsertionResultNullifier sequential_nullifier_to_wire( - const world_state::SequentialInsertionResult& d) + const crypto::merkle_tree::SequentialInsertionResult& d) { return { .lowLeafWitnessData = vec_to_wire(d.low_leaf_witness_data, nullifier_witness_to_wire), @@ -343,8 +348,8 @@ inline wire::SequentialInsertionResultNullifier sequential_nullifier_to_wire( nullifier_witness_to_wire) }; } -inline world_state::SequentialInsertionResult sequential_nullifier_from_wire( - const wire::SequentialInsertionResultNullifier& w) +inline crypto::merkle_tree::SequentialInsertionResult +sequential_nullifier_from_wire(const wire::SequentialInsertionResultNullifier& w) { return { .low_leaf_witness_data = vec_from_wire>( @@ -354,151 +359,4 @@ inline world_state::SequentialInsertionResult state_reference_to_wire(const world_state::StateReference& d) -{ - std::vector r; - r.reserve(d.size()); - for (const auto& [tree_id, tree_ref] : d) { - r.push_back( - { .treeId = tree_id_to_wire(tree_id), .root = fr_to_wire(tree_ref.first), .size = tree_ref.second }); - } - return r; -} - -inline world_state::StateReference state_reference_from_wire(const std::vector& w) -{ - world_state::StateReference r; - r.reserve(w.size()); - for (const auto& entry : w) { - r.emplace(tree_id_from_wire(entry.treeId), - world_state::TreeStateReference{ fr_from_wire(entry.root), entry.size }); - } - return r; -} - -inline wire::DBStats db_stats_to_wire(const bb::lmdblib::DBStats& d) -{ - return { .name = d.name, .numDataItems = d.numDataItems, .totalUsedSize = d.totalUsedSize }; -} - -inline bb::lmdblib::DBStats db_stats_from_wire(const wire::DBStats& w) -{ - return bb::lmdblib::DBStats(w.name, w.numDataItems, w.totalUsedSize); -} - -inline wire::TreeDBStats tree_db_stats_to_wire(const bb::crypto::merkle_tree::TreeDBStats& d) -{ - return { .mapSize = d.mapSize, - .physicalFileSize = d.physicalFileSize, - .blocksDBStats = db_stats_to_wire(d.blocksDBStats), - .nodesDBStats = db_stats_to_wire(d.nodesDBStats), - .leafPreimagesDBStats = db_stats_to_wire(d.leafPreimagesDBStats), - .leafIndicesDBStats = db_stats_to_wire(d.leafIndicesDBStats), - .blockIndicesDBStats = db_stats_to_wire(d.blockIndicesDBStats) }; -} - -inline bb::crypto::merkle_tree::TreeDBStats tree_db_stats_from_wire(const wire::TreeDBStats& w) -{ - return { w.mapSize, - w.physicalFileSize, - db_stats_from_wire(w.blocksDBStats), - db_stats_from_wire(w.nodesDBStats), - db_stats_from_wire(w.leafPreimagesDBStats), - db_stats_from_wire(w.leafIndicesDBStats), - db_stats_from_wire(w.blockIndicesDBStats) }; -} - -inline wire::TreeMeta tree_meta_to_wire(const bb::crypto::merkle_tree::TreeMeta& d) -{ - return { .name = d.name, - .depth = d.depth, - .size = d.size, - .committedSize = d.committedSize, - .root = fr_to_wire(d.root), - .initialSize = d.initialSize, - .initialRoot = fr_to_wire(d.initialRoot), - .oldestHistoricBlock = d.oldestHistoricBlock, - .unfinalizedBlockHeight = d.unfinalizedBlockHeight, - .finalizedBlockHeight = d.finalizedBlockHeight }; -} - -inline bb::crypto::merkle_tree::TreeMeta tree_meta_from_wire(const wire::TreeMeta& w) -{ - return { w.name, - w.depth, - w.size, - w.committedSize, - fr_from_wire(w.root), - w.initialSize, - fr_from_wire(w.initialRoot), - w.oldestHistoricBlock, - w.unfinalizedBlockHeight, - w.finalizedBlockHeight }; -} - -inline wire::WorldStateDBStats world_state_db_stats_to_wire(const bb::world_state::WorldStateDBStats& d) -{ - return { .noteHashTreeStats = tree_db_stats_to_wire(d.noteHashTreeStats), - .messageTreeStats = tree_db_stats_to_wire(d.messageTreeStats), - .archiveTreeStats = tree_db_stats_to_wire(d.archiveTreeStats), - .publicDataTreeStats = tree_db_stats_to_wire(d.publicDataTreeStats), - .nullifierTreeStats = tree_db_stats_to_wire(d.nullifierTreeStats) }; -} - -inline bb::world_state::WorldStateDBStats world_state_db_stats_from_wire(const wire::WorldStateDBStats& w) -{ - return { tree_db_stats_from_wire(w.noteHashTreeStats), - tree_db_stats_from_wire(w.messageTreeStats), - tree_db_stats_from_wire(w.archiveTreeStats), - tree_db_stats_from_wire(w.publicDataTreeStats), - tree_db_stats_from_wire(w.nullifierTreeStats) }; -} - -inline wire::WorldStateMeta world_state_meta_to_wire(const bb::world_state::WorldStateMeta& d) -{ - return { .noteHashTreeMeta = tree_meta_to_wire(d.noteHashTreeMeta), - .messageTreeMeta = tree_meta_to_wire(d.messageTreeMeta), - .archiveTreeMeta = tree_meta_to_wire(d.archiveTreeMeta), - .publicDataTreeMeta = tree_meta_to_wire(d.publicDataTreeMeta), - .nullifierTreeMeta = tree_meta_to_wire(d.nullifierTreeMeta) }; -} - -inline bb::world_state::WorldStateMeta world_state_meta_from_wire(const wire::WorldStateMeta& w) -{ - return { tree_meta_from_wire(w.noteHashTreeMeta), - tree_meta_from_wire(w.messageTreeMeta), - tree_meta_from_wire(w.archiveTreeMeta), - tree_meta_from_wire(w.publicDataTreeMeta), - tree_meta_from_wire(w.nullifierTreeMeta) }; -} - -inline wire::WorldStateStatusSummary world_state_status_summary_to_wire( - const bb::world_state::WorldStateStatusSummary& d) -{ - return { .unfinalizedBlockNumber = d.unfinalizedBlockNumber, - .finalizedBlockNumber = d.finalizedBlockNumber, - .oldestHistoricalBlock = d.oldestHistoricalBlock, - .treesAreSynched = d.treesAreSynched }; -} - -inline bb::world_state::WorldStateStatusSummary world_state_status_summary_from_wire( - const wire::WorldStateStatusSummary& w) -{ - return { w.unfinalizedBlockNumber, w.finalizedBlockNumber, w.oldestHistoricalBlock, w.treesAreSynched }; -} - -inline wire::WorldStateStatusFull world_state_status_full_to_wire(const bb::world_state::WorldStateStatusFull& d) -{ - return { .summary = world_state_status_summary_to_wire(d.summary), - .dbStats = world_state_db_stats_to_wire(d.dbStats), - .meta = world_state_meta_to_wire(d.meta) }; -} - -inline bb::world_state::WorldStateStatusFull world_state_status_full_from_wire(const wire::WorldStateStatusFull& w) -{ - return { world_state_status_summary_from_wire(w.summary), - world_state_db_stats_from_wire(w.dbStats), - world_state_meta_from_wire(w.meta) }; -} - } // namespace bb::wsdb diff --git a/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt deleted file mode 100644 index a44babcef490..000000000000 --- a/barretenberg/cpp/src/barretenberg/world_state/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -barretenberg_module(world_state crypto_merkle_tree crypto_poseidon2) - -# world_state needs -fPIC because it's linked into the nodejs_module shared library -set_target_properties(world_state_objects PROPERTIES POSITION_INDEPENDENT_CODE ON) diff --git a/barretenberg/cpp/src/barretenberg/wsdb/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/wsdb/CMakeLists.txt deleted file mode 100644 index ca88f8e4b9ab..000000000000 --- a/barretenberg/cpp/src/barretenberg/wsdb/CMakeLists.txt +++ /dev/null @@ -1,88 +0,0 @@ -# IPC client library (used by AVM simulator to talk to aztec-wsdb). -# Sources are generated by ipc-codegen from wsdb_schema.jsonc. The custom -# command below wires generation into ninja so clients and servers only -# consume the generated output directory. -set(WSDB_SCHEMA ${CMAKE_CURRENT_SOURCE_DIR}/wsdb_schema.jsonc) -set(WSDB_GEN_PARENT ${CMAKE_CURRENT_SOURCE_DIR}) -set(WSDB_GEN_DIR ${WSDB_GEN_PARENT}/generated) -set(WSDB_GEN_OUTPUTS - ${WSDB_GEN_DIR}/wsdb_ipc_client.cpp - ${WSDB_GEN_DIR}/wsdb_ipc_client.hpp - ${WSDB_GEN_DIR}/wsdb_ipc_server.hpp - ${WSDB_GEN_DIR}/wsdb_types.hpp - ${WSDB_GEN_DIR}/ipc_codegen/msgpack_adaptor.hpp - ${WSDB_GEN_DIR}/ipc_codegen/named_union.hpp - ${WSDB_GEN_DIR}/ipc_codegen/throw.hpp -) -set(IPC_CODEGEN_DIR ${CMAKE_SOURCE_DIR}/../../ipc-codegen) -file(GLOB_RECURSE IPC_CODEGEN_SRC - ${IPC_CODEGEN_DIR}/src/*.ts - ${IPC_CODEGEN_DIR}/templates/cpp/*.hpp -) -add_custom_command( - OUTPUT ${WSDB_GEN_OUTPUTS} - COMMAND node --experimental-strip-types --experimental-transform-types --no-warnings - ${IPC_CODEGEN_DIR}/src/generate.ts - --schema ${WSDB_SCHEMA} - --lang cpp - --out ${WSDB_GEN_DIR} - --client --server - --cpp-namespace bb::wsdb - --cpp-include-dir barretenberg/wsdb/generated - DEPENDS ${WSDB_SCHEMA} ${IPC_CODEGEN_SRC} - COMMENT "Generating WSDB IPC client + server from wsdb_schema.jsonc" - VERBATIM -) -add_custom_target(wsdb_ipc_generated DEPENDS ${WSDB_GEN_OUTPUTS}) -add_library( - wsdb_ipc_client - STATIC - ${WSDB_GEN_DIR}/wsdb_ipc_client.cpp -) -add_dependencies(wsdb_ipc_client wsdb_ipc_generated) -target_include_directories( - wsdb_ipc_client - PUBLIC - ${WSDB_GEN_DIR} -) -target_link_libraries( - wsdb_ipc_client - PUBLIC - barretenberg - ipc_runtime -) - -# aztec-wsdb binary (standalone world state database server) -add_executable( - aztec-wsdb - main.cpp - cli.cpp - wsdb_handlers.cpp - wsdb_ipc_server.cpp -) -add_dependencies(aztec-wsdb wsdb_ipc_generated) -target_include_directories( - aztec-wsdb - PRIVATE - ${WSDB_GEN_DIR} -) -target_link_libraries( - aztec-wsdb - PRIVATE - barretenberg - world_state - ipc_runtime - env -) -if(ENABLE_STACKTRACES) - target_link_libraries( - aztec-wsdb - PUBLIC - Backward::Interface - ) - target_link_options( - aztec-wsdb - PRIVATE - -ldw -lelf - ) -endif() diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_client.hpp b/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_client.hpp deleted file mode 100644 index ed738a4c419b..000000000000 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_client.hpp +++ /dev/null @@ -1,3 +0,0 @@ -#pragma once - -#include "barretenberg/wsdb/generated/wsdb_ipc_client.hpp" diff --git a/bootstrap.sh b/bootstrap.sh index 36abba98e01c..d6f8f522fe62 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -562,7 +562,9 @@ function release { projects=( barretenberg/cpp ipc-runtime - wsdb + native-packages/lmdblib + native-packages/kvdb + native-packages/wsdb barretenberg/ts barretenberg/rust noir diff --git a/native-packages/.clang-format b/native-packages/.clang-format new file mode 100644 index 000000000000..14a6b3bac238 --- /dev/null +++ b/native-packages/.clang-format @@ -0,0 +1,26 @@ +PointerAlignment: Left +ColumnLimit: 120 +IndentWidth: 4 +BinPackArguments: false +BinPackParameters: false +Cpp11BracedListStyle: false +AlwaysBreakAfterReturnType: None +AlwaysBreakAfterDefinitionReturnType: None +PenaltyReturnTypeOnItsOwnLine: 1000000 +BreakConstructorInitializers: BeforeComma +BreakBeforeBraces: Custom +BraceWrapping: + AfterClass: false + AfterEnum: false + AfterFunction: true + AfterNamespace: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + SplitEmptyFunction: false + SplitEmptyRecord: false + SplitEmptyNamespace: false +AllowShortFunctionsOnASingleLine : Inline +SortIncludes: true diff --git a/wsdb/.gitignore b/native-packages/kvdb/.gitignore similarity index 63% rename from wsdb/.gitignore rename to native-packages/kvdb/.gitignore index 5e538a0ae7de..a0e6495b6ebd 100644 --- a/wsdb/.gitignore +++ b/native-packages/kvdb/.gitignore @@ -1,9 +1,13 @@ -ts/ +cpp/build/ +ts/dest/ +ts/build/ +ts/packages/ +ts/.tsbuildinfo +node_modules +*.log .yarn/* !.yarn/patches !.yarn/plugins !.yarn/releases !.yarn/sdks !.yarn/versions -node_modules -*.log diff --git a/native-packages/kvdb/.rebuild_patterns b/native-packages/kvdb/.rebuild_patterns new file mode 100644 index 000000000000..9cb9042b35bc --- /dev/null +++ b/native-packages/kvdb/.rebuild_patterns @@ -0,0 +1,6 @@ +^native-packages/kvdb/bootstrap\.sh$ +^native-packages/kvdb/package\.json$ +^native-packages/kvdb/\.rebuild_patterns$ +^native-packages/kvdb/cpp/ +^native-packages/kvdb/ts/src/ +^native-packages/lmdblib/ diff --git a/wsdb/.yarnrc.yml b/native-packages/kvdb/.yarnrc.yml similarity index 100% rename from wsdb/.yarnrc.yml rename to native-packages/kvdb/.yarnrc.yml diff --git a/native-packages/kvdb/bootstrap.sh b/native-packages/kvdb/bootstrap.sh new file mode 100755 index 000000000000..179cc2a58f38 --- /dev/null +++ b/native-packages/kvdb/bootstrap.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +source $(git rev-parse --show-toplevel)/ci3/source_bootstrap + +ROOT=$(git rev-parse --show-toplevel) +PKG="$ROOT/native-packages/kvdb" +NAPI_BINARY=nodejs_module.node + +hash=$(hash_str \ + $(../lmdblib/bootstrap.sh hash) \ + $(cache_content_hash .rebuild_patterns)) + +# Build the NAPI addon. kvdb is barretenberg-free: it links only the lmdblib +# sibling package's archives (+ lmdb, node-addon-api). lmdblib is assumed prebuilt. +function build_native { + local build_dir="cpp/build" + CC=$(which clang) CXX=$(which clang++) cmake -S cpp -B "$build_dir" -G Ninja >/dev/null + cmake --build "$build_dir" --target nodejs_module + local target_dir="ts/build/$(arch)-$(os)" + mkdir -p "$target_dir" + cp "$build_dir/lib/$NAPI_BINARY" "$target_dir/$NAPI_BINARY" +} + +function build { + echo_header "kvdb build" + build_native + # Create the per-arch workspace packages BEFORE installing: npm_install_deps does + # a clean `yarn install --immutable` on a cache miss, and the @aztec/kvdb- + # optionalDependencies must resolve to these local workspaces (not npm) or the + # install 404s. + (cd ts && ./scripts/prepare_arch_packages.sh "$(arch)-$(os)=build/$(arch)-$(os)/$NAPI_BINARY") + npm_install_deps + yarn build +} + +function clean { + rm -rf ts/dest ts/build ts/packages ts/node_modules node_modules cpp/build +} + +function release { + build_native + npm_install_deps + yarn build + (cd ts && ./scripts/prepare_arch_packages.sh) + for package_dir in ts/packages/*; do + (cd "$package_dir" && retry "deploy_npm ${REF_NAME#v}") + done + (cd ts && retry "deploy_npm ${REF_NAME#v}") +} + +export -f build_native build clean release + +case "$cmd" in + "") + build + ;; + "hash") + echo "$hash" + ;; + *) + default_cmd_handler "$@" + ;; +esac diff --git a/native-packages/kvdb/cpp/.gitignore b/native-packages/kvdb/cpp/.gitignore new file mode 100644 index 000000000000..61c3bc75a05e --- /dev/null +++ b/native-packages/kvdb/cpp/.gitignore @@ -0,0 +1 @@ +.yarn diff --git a/native-packages/kvdb/cpp/.yarnrc.yml b/native-packages/kvdb/cpp/.yarnrc.yml new file mode 100644 index 000000000000..1d8b263ddcf5 --- /dev/null +++ b/native-packages/kvdb/cpp/.yarnrc.yml @@ -0,0 +1,5 @@ +nodeLinker: node-modules + +npmMinimalAgeGate: 7d + +npmPreapprovedPackages: [] diff --git a/native-packages/kvdb/cpp/CMakeLists.txt b/native-packages/kvdb/cpp/CMakeLists.txt new file mode 100644 index 000000000000..87214379e60f --- /dev/null +++ b/native-packages/kvdb/cpp/CMakeLists.txt @@ -0,0 +1,112 @@ +# Standalone build for the kvdb Node NAPI addon (nodejs_module.node). +# +# This package owns the kv-store native backend (the LMDBStore NAPI wrapper) that +# barretenberg no longer hosts. It is barretenberg-free: it links only the lmdblib +# sibling package's archives + lmdb + the Node addon runtime. It uses msgpack-c +# directly (no bb serialization framework) and vendors the small message header / +# dispatcher it needs under src/messaging. + +cmake_minimum_required(VERSION 3.24) +project(aztec_kvdb CXX C) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) + +# Match barretenberg's prebuilt archive (Release -> -O3 -DNDEBUG): bb headers compiled +# here must use the same NDEBUG/opt as bb to avoid ODR-divergent inlined code. +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif() + +add_definitions(-DNAPI_VERSION=9) + +# --------------------------------------------------------------------------- +# lmdblib sibling package build outputs (the only native dependency). +# --------------------------------------------------------------------------- +get_filename_component(REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../.." ABSOLUTE) +set(LMDBLIB_PKG "${REPO_ROOT}/native-packages/lmdblib") +set(LMDBLIB_SRC "${LMDBLIB_PKG}/cpp/src") +set(LMDBLIB_LIB "${LMDBLIB_PKG}/cpp/build/lib/liblmdblib.a") +set(LMDB_INCLUDE "${LMDBLIB_PKG}/cpp/build/_deps/lmdb/src/lmdb_repo/libraries/liblmdb") +set(LMDB_LIB "${LMDB_INCLUDE}/liblmdb.a") + +foreach(artifact "${LMDBLIB_LIB}" "${LMDB_LIB}") + if(NOT EXISTS "${artifact}") + message(FATAL_ERROR "Missing prebuilt artifact: ${artifact}\n" + "Build lmdblib first (cd native-packages/lmdblib && ./bootstrap.sh).") + endif() +endforeach() + +# --------------------------------------------------------------------------- +# msgpack-c headers (own copy; commit must match lmdblib's so the wire ABI lines up). +# --------------------------------------------------------------------------- +set(DEPS_PREFIX "${CMAKE_BINARY_DIR}/_deps") +include(ExternalProject) + +set(MSGPACK_PREFIX "${DEPS_PREFIX}/msgpack-c") +set(MSGPACK_INCLUDE "${MSGPACK_PREFIX}/src/msgpack-c/include") +ExternalProject_Add( + msgpack-c + PREFIX ${MSGPACK_PREFIX} + DOWNLOAD_COMMAND + sh -c "mkdir -p ${MSGPACK_PREFIX}/src/msgpack-c && cd ${MSGPACK_PREFIX}/src/msgpack-c && git init --quiet && (git remote add origin https://github.com/AztecProtocol/msgpack-c.git 2>/dev/null || true) && git fetch --depth 1 origin --quiet c0334576ed657fb3b3c49e8e61402989fb84146d && git reset --quiet --hard FETCH_HEAD" + SOURCE_DIR ${MSGPACK_PREFIX}/src/msgpack-c + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + UPDATE_COMMAND "" +) + +# --------------------------------------------------------------------------- +# Node addon-api / node-api headers (resolved from this package's node_modules). +# --------------------------------------------------------------------------- +execute_process( + COMMAND yarn --immutable + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) +execute_process( + COMMAND node -p "require('node-addon-api').include" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_VARIABLE NODE_ADDON_API_DIR +) +execute_process( + COMMAND node -p "require('node-api-headers').include_dir" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_VARIABLE NODE_API_HEADERS_DIR +) +string(REGEX REPLACE "[\r\n\"]" "" NODE_ADDON_API_DIR ${NODE_ADDON_API_DIR}) +string(REGEX REPLACE "[\r\n\"]" "" NODE_API_HEADERS_DIR ${NODE_API_HEADERS_DIR}) + +# libc++/-march=skylake match lmdblib's archive (ABI + arch compatibility). Static +# libc++/libgcc keep the addon self-contained (no libc++.so.1 runtime dependency). +add_compile_definitions(MSGPACK_NO_BOOST MSGPACK_USE_STD_VARIANT_ADAPTOR NO_PAR_ALGOS) +add_compile_options(-march=skylake -fconstexpr-steps=100000000 -fbracket-depth=1024) +add_compile_options(-Wno-deprecated-declarations -Wno-missing-field-initializers) +add_compile_options(-stdlib=libc++) +add_link_options(-stdlib=libc++ -static-libstdc++ -static-libgcc) + +include_directories( + ${CMAKE_CURRENT_SOURCE_DIR}/src + ${LMDBLIB_SRC} + ${MSGPACK_INCLUDE} + ${LMDB_INCLUDE} +) + +set(KVDB_SOURCES + src/init_module.cpp + src/lmdb_store/lmdb_store_wrapper.cpp + src/util/promise.cpp +) +add_library(nodejs_module SHARED ${KVDB_SOURCES}) +add_dependencies(nodejs_module msgpack-c) +set_target_properties(nodejs_module PROPERTIES PREFIX "" SUFFIX ".node") +target_include_directories(nodejs_module PRIVATE ${NODE_API_HEADERS_DIR} ${NODE_ADDON_API_DIR}) + +target_link_libraries(nodejs_module PRIVATE ${LMDBLIB_LIB} ${LMDB_LIB} pthread) + +# On macOS, Node.js N-API symbols are provided by the runtime, not at link time. +if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + target_link_options(nodejs_module PRIVATE "-undefined" "dynamic_lookup") +endif() diff --git a/native-packages/kvdb/cpp/package.json b/native-packages/kvdb/cpp/package.json new file mode 100644 index 000000000000..b30468a65e83 --- /dev/null +++ b/native-packages/kvdb/cpp/package.json @@ -0,0 +1,15 @@ +{ + "name": "nodejs_module", + "private": true, + "version": "0.0.0", + "packageManager": "yarn@4.13.0", + "dependencies": { + "node-addon-api": "^8.0.0", + "node-api-headers": "^1.1.0" + }, + "binary": { + "napi_versions": [ + 9 + ] + } +} diff --git a/native-packages/kvdb/cpp/src/init_module.cpp b/native-packages/kvdb/cpp/src/init_module.cpp new file mode 100644 index 000000000000..11f8d04c5295 --- /dev/null +++ b/native-packages/kvdb/cpp/src/init_module.cpp @@ -0,0 +1,11 @@ +#include "lmdb_store/lmdb_store_wrapper.hpp" +#include "napi.h" + +Napi::Object Init(Napi::Env env, Napi::Object exports) { + exports.Set(Napi::String::New(env, "LMDBStore"), + bb::nodejs::lmdb_store::LMDBStoreWrapper::get_class(env)); + return exports; +} + +// NOLINTNEXTLINE +NODE_API_MODULE(addon, Init) diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/lmdb_store/lmdb_store_message.hpp b/native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_message.hpp similarity index 70% rename from barretenberg/cpp/src/barretenberg/nodejs_module/lmdb_store/lmdb_store_message.hpp rename to native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_message.hpp index b63bd1849f7d..21d49f57bf32 100644 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/lmdb_store/lmdb_store_message.hpp +++ b/native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_message.hpp @@ -1,7 +1,7 @@ #pragma once -#include "barretenberg/lmdblib/types.hpp" -#include "barretenberg/messaging/header.hpp" -#include "barretenberg/serialize/msgpack.hpp" +#include "lmdblib/types.hpp" +#include "messaging/header.hpp" +#include "serialization.hpp" #include "msgpack/adaptor/define_decl.hpp" #include #include @@ -33,43 +33,43 @@ enum LMDBStoreMessageType { struct OpenDatabaseRequest { std::string db; std::optional uniqueKeys; - SERIALIZATION_FIELDS(db, uniqueKeys); + MSGPACK_DEFINE_MAP(db, uniqueKeys); }; struct GetRequest { lmdblib::KeysVector keys; std::string db; - SERIALIZATION_FIELDS(keys, db); + MSGPACK_DEFINE_MAP(keys, db); }; struct GetResponse { lmdblib::OptionalValuesVector values; - SERIALIZATION_FIELDS(values); + MSGPACK_DEFINE_MAP(values); }; struct HasRequest { // std::map> entries; lmdblib::KeyOptionalValuesVector entries; std::string db; - SERIALIZATION_FIELDS(entries, db); + MSGPACK_DEFINE_MAP(entries, db); }; struct HasResponse { // std::map exists; std::vector exists; - SERIALIZATION_FIELDS(exists); + MSGPACK_DEFINE_MAP(exists); }; struct Batch { lmdblib::KeyDupValuesVector addEntries; lmdblib::KeyOptionalValuesVector removeEntries; - SERIALIZATION_FIELDS(addEntries, removeEntries); + MSGPACK_DEFINE_MAP(addEntries, removeEntries); }; struct BatchRequest { std::map batches; - SERIALIZATION_FIELDS(batches); + MSGPACK_DEFINE_MAP(batches); }; struct StartCursorRequest { @@ -78,65 +78,65 @@ struct StartCursorRequest { std::optional count; std::optional onePage; std::string db; - SERIALIZATION_FIELDS(key, reverse, count, onePage, db); + MSGPACK_DEFINE_MAP(key, reverse, count, onePage, db); }; struct StartCursorResponse { std::optional cursor; lmdblib::KeyDupValuesVector entries; - SERIALIZATION_FIELDS(cursor, entries); + MSGPACK_DEFINE_MAP(cursor, entries); }; struct AdvanceCursorRequest { uint64_t cursor; std::optional count; - SERIALIZATION_FIELDS(cursor, count); + MSGPACK_DEFINE_MAP(cursor, count); }; struct AdvanceCursorCountRequest { uint64_t cursor; lmdblib::Key endKey; - SERIALIZATION_FIELDS(cursor, endKey); + MSGPACK_DEFINE_MAP(cursor, endKey); }; struct CloseCursorRequest { uint64_t cursor; - SERIALIZATION_FIELDS(cursor); + MSGPACK_DEFINE_MAP(cursor); }; struct AdvanceCursorResponse { lmdblib::KeyDupValuesVector entries; bool done; - SERIALIZATION_FIELDS(entries, done); + MSGPACK_DEFINE_MAP(entries, done); }; struct AdvanceCursorCountResponse { uint64_t count; bool done; - SERIALIZATION_FIELDS(count, done); + MSGPACK_DEFINE_MAP(count, done); }; struct BoolResponse { bool ok; - SERIALIZATION_FIELDS(ok); + MSGPACK_DEFINE_MAP(ok); }; struct BatchResponse { uint64_t durationNs; - SERIALIZATION_FIELDS(durationNs); + MSGPACK_DEFINE_MAP(durationNs); }; struct StatsResponse { std::vector stats; uint64_t dbMapSizeBytes; uint64_t dbPhysicalFileSizeBytes; - SERIALIZATION_FIELDS(stats, dbMapSizeBytes, dbPhysicalFileSizeBytes); + MSGPACK_DEFINE_MAP(stats, dbMapSizeBytes, dbPhysicalFileSizeBytes); }; struct CopyStoreRequest { std::string dstPath; std::optional compact; - SERIALIZATION_FIELDS(dstPath, compact); + MSGPACK_DEFINE_MAP(dstPath, compact); }; } // namespace bb::nodejs::lmdb_store diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/lmdb_store/lmdb_store_wrapper.cpp b/native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_wrapper.cpp similarity index 97% rename from barretenberg/cpp/src/barretenberg/nodejs_module/lmdb_store/lmdb_store_wrapper.cpp rename to native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_wrapper.cpp index b891f0e3f115..cb55b75895db 100644 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/lmdb_store/lmdb_store_wrapper.cpp +++ b/native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_wrapper.cpp @@ -1,7 +1,7 @@ -#include "barretenberg/nodejs_module/lmdb_store/lmdb_store_wrapper.hpp" -#include "barretenberg/lmdblib/lmdb_store.hpp" -#include "barretenberg/lmdblib/types.hpp" -#include "barretenberg/nodejs_module/lmdb_store/lmdb_store_message.hpp" +#include "lmdb_store/lmdb_store_wrapper.hpp" +#include "lmdblib/lmdb_store.hpp" +#include "lmdblib/types.hpp" +#include "lmdb_store/lmdb_store_message.hpp" #include "napi.h" #include #include @@ -113,7 +113,7 @@ void LMDBStoreWrapper::verify_store() const if (_store) { return; } - throw std::runtime_error(format("LMDB store unavailable, was close already called?")); + throw std::runtime_error(bb::lmdblib::format("LMDB store unavailable, was close already called?")); } BoolResponse LMDBStoreWrapper::open_database(const OpenDatabaseRequest& req) diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/lmdb_store/lmdb_store_wrapper.hpp b/native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_wrapper.hpp similarity index 85% rename from barretenberg/cpp/src/barretenberg/nodejs_module/lmdb_store/lmdb_store_wrapper.hpp rename to native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_wrapper.hpp index 4fea656edc5b..2f248efe73cc 100644 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/lmdb_store/lmdb_store_wrapper.hpp +++ b/native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_wrapper.hpp @@ -1,12 +1,12 @@ #pragma once -#include "barretenberg/lmdblib/lmdb_cursor.hpp" -#include "barretenberg/lmdblib/lmdb_store.hpp" -#include "barretenberg/lmdblib/types.hpp" -#include "barretenberg/messaging/dispatcher.hpp" -#include "barretenberg/messaging/header.hpp" -#include "barretenberg/nodejs_module/lmdb_store/lmdb_store_message.hpp" -#include "barretenberg/nodejs_module/util/message_processor.hpp" +#include "lmdblib/lmdb_cursor.hpp" +#include "lmdblib/lmdb_store.hpp" +#include "lmdblib/types.hpp" +#include "messaging/dispatcher.hpp" +#include "messaging/header.hpp" +#include "lmdb_store/lmdb_store_message.hpp" +#include "util/message_processor.hpp" #include #include #include diff --git a/native-packages/kvdb/cpp/src/messaging/dispatcher.hpp b/native-packages/kvdb/cpp/src/messaging/dispatcher.hpp new file mode 100644 index 000000000000..374b1a00b5ad --- /dev/null +++ b/native-packages/kvdb/cpp/src/messaging/dispatcher.hpp @@ -0,0 +1,59 @@ +#pragma once + +#include "messaging/header.hpp" +#include "serialization.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace bb::messaging { + +using message_handler = + std::function; +struct MessageHandler { + bool unique; + message_handler handler; +}; + +class MessageDispatcher { +private: + std::unordered_map message_handlers; + mutable std::shared_mutex mutex; + +public: + MessageDispatcher() = default; + + bool on_new_data(msgpack::object &obj, msgpack::sbuffer &buffer) const { + bb::messaging::HeaderOnlyMessage header; + obj.convert(header); + + auto iter = message_handlers.find(header.msgType); + if (iter == message_handlers.end()) { + throw std::runtime_error("No registered handler for message of type " + + std::to_string(header.msgType)); + } + + // If the msg type has been marked as 'unique' then we need to give it + // exclusive execution context + if (iter->second.unique) { + std::unique_lock lock(mutex); + return (iter->second.handler)(obj, buffer); + } + std::shared_lock lock(mutex); + return (iter->second.handler)(obj, buffer); + } + + void register_target(uint32_t msgType, const message_handler &handler, + bool unique = false) { + MessageHandler msg_handler{unique, handler}; + message_handlers.insert({msgType, msg_handler}); + } +}; + +} // namespace bb::messaging diff --git a/native-packages/kvdb/cpp/src/messaging/header.hpp b/native-packages/kvdb/cpp/src/messaging/header.hpp new file mode 100644 index 000000000000..358079b87798 --- /dev/null +++ b/native-packages/kvdb/cpp/src/messaging/header.hpp @@ -0,0 +1,52 @@ +#pragma once +#include "serialization.hpp" +#include +#include + +namespace bb::messaging { + +enum SystemMsgTypes { TERMINATE = 0, PING = 1, PONG = 2 }; + +const uint32_t FIRST_APP_MSG_TYPE = 100; + +struct MsgHeader { + uint32_t messageId; // Unique Id for the message + uint32_t + requestId; // Id of the message this is responding too (may not be used) + + MSGPACK_DEFINE_MAP(messageId, requestId); + + MsgHeader() = default; + + MsgHeader(uint32_t reqId) : requestId(reqId) {} + + MsgHeader(uint32_t msgId, uint32_t reqId) + : messageId(msgId), requestId(reqId) {} +}; + +struct HeaderOnlyMessage { + uint32_t msgType; + MsgHeader header; + + HeaderOnlyMessage(uint32_t type, MsgHeader &hdr) + : msgType(type), header(hdr) {} + + HeaderOnlyMessage() = default; + + MSGPACK_DEFINE_MAP(msgType, header); +}; + +template struct TypedMessage { + uint32_t msgType; + MsgHeader header; + T value; + + TypedMessage(uint32_t type, MsgHeader &hdr, const T &val) + : msgType(type), header(hdr), value(val) {} + + TypedMessage() = default; + + MSGPACK_DEFINE_MAP(msgType, header, value); +}; + +} // namespace bb::messaging diff --git a/native-packages/kvdb/cpp/src/serialization.hpp b/native-packages/kvdb/cpp/src/serialization.hpp new file mode 100644 index 000000000000..e238db4d16d4 --- /dev/null +++ b/native-packages/kvdb/cpp/src/serialization.hpp @@ -0,0 +1,14 @@ +#pragma once + +// msgpack-c entrypoint for kvdb. We use msgpack-c directly (no barretenberg +// serialization framework). The AztecProtocol msgpack-c fork expects the +// includer to provide THROW/RETHROW macros; define them to standard exception +// handling. +#ifndef THROW +#define THROW throw +#endif +#ifndef RETHROW +#define RETHROW throw +#endif + +#include diff --git a/native-packages/kvdb/cpp/src/util/async_op.hpp b/native-packages/kvdb/cpp/src/util/async_op.hpp new file mode 100644 index 000000000000..bb1494d5e428 --- /dev/null +++ b/native-packages/kvdb/cpp/src/util/async_op.hpp @@ -0,0 +1,200 @@ +#pragma once + +#include "serialization.hpp" +#include +#include +#include +#include +#include + +#ifndef _WIN32 +#include +#endif + +namespace bb::nodejs { + +using async_fn = std::function; + +/** + * @brief Encapsulatest some work that can be done off the JavaScript main thread + * + * This class takes a Deferred instance (i.e. a Promise to JS), execute some work in a separate thread, and then report + * back on the result. The async execution _must not_ touch the JS environment. Everything that's needed to complete the + * work must be copied into memory owned by the C++ code. The same has to be done when reporting back the result: keep + * the result in memory owned by the C++ code and copy it back to the JS environment in the OnOK/OnError methods. + * + * OnOK/OnError will be called on the main JS thread, so it's safe to interact with the JS environment there. + * + * Instances of this class are managed by the NodeJS environment and execute on a libuv thread. + * Docs + * . - https://github.com/nodejs/node-addon-api/blob/cc06369aa4dd29e585600b8b47839c1297df962d/doc/async_worker.md + * . - https://nodejs.github.io/node-addon-examples/special-topics/asyncworker + */ +class AsyncOperation : public Napi::AsyncWorker { + public: + AsyncOperation(Napi::Env env, std::shared_ptr deferred, async_fn fn) + : Napi::AsyncWorker(env) + , _fn(std::move(fn)) + , _deferred(std::move(deferred)) + {} + + AsyncOperation(const AsyncOperation&) = delete; + AsyncOperation& operator=(const AsyncOperation&) = delete; + AsyncOperation(AsyncOperation&&) = delete; + AsyncOperation& operator=(AsyncOperation&&) = delete; + + ~AsyncOperation() override = default; + + void Execute() override + { + try { + _fn(_result); + } catch (const std::exception& e) { + SetError(e.what()); + } catch (...) { + // Catch any other exception type that's not derived from std::exception + // This ensures the promise is always rejected rather than leaving it hanging + SetError("Unknown exception occurred during async operation"); + } + } + + void OnOK() override + { + auto buf = Napi::Buffer::Copy(Env(), _result.data(), _result.size()); + _deferred->Resolve(buf); + } + void OnError(const Napi::Error& e) override { _deferred->Reject(e.Value()); } + + private: + async_fn _fn; + std::shared_ptr _deferred; + msgpack::sbuffer _result; +}; + +/** + * @brief Runs work on a dedicated std::thread instead of the libuv thread pool. + * + * Unlike AsyncOperation (which uses Napi::AsyncWorker and occupies a libuv thread), + * this class spawns a new OS thread for each operation. This prevents AVM simulations + * from exhausting the libuv thread pool, which would deadlock when C++ callbacks need + * to invoke JS functions that themselves require libuv threads (e.g., LMDB reads). + * + * The completion callback (resolve/reject) is posted back to the JS main thread via + * a Napi::ThreadSafeFunction, so the event loop returns immediately after launch + * and is woken up only when the work is done. + * + * Prevent use-after-free: the TSFN callback runs asynchronously on the JS thread + * (napi_tsfn_blocking only blocks on queue insertion, NOT on callback completion). + * Both the worker thread lambda and the callback capture a shared_ptr to keep the + * object alive until both are done. + * + * Usage: `ThreadedAsyncOperation::Run(env, deferred, fn);` + */ +class ThreadedAsyncOperation : public std::enable_shared_from_this { + public: + ThreadedAsyncOperation(Napi::Env env, std::shared_ptr deferred, async_fn fn) + : _fn(std::move(fn)) + , _deferred(std::move(deferred)) + { + auto dummy = Napi::Function::New(env, [](const Napi::CallbackInfo&) {}); + _completion_tsfn = Napi::ThreadSafeFunction::New(env, dummy, "ThreadedAsyncOpComplete", 0, 1); + } + + ThreadedAsyncOperation(const ThreadedAsyncOperation&) = delete; + ThreadedAsyncOperation& operator=(const ThreadedAsyncOperation&) = delete; + ThreadedAsyncOperation(ThreadedAsyncOperation&&) = delete; + ThreadedAsyncOperation& operator=(ThreadedAsyncOperation&&) = delete; + + ~ThreadedAsyncOperation() = default; + + static void Run(Napi::Env env, std::shared_ptr deferred, async_fn fn) + { + auto op = std::make_shared(env, std::move(deferred), std::move(fn)); + op->Queue(); + } + + private: + // AVM simulation call chains are deep. Non-main threads get a 512 KB default stack on + // macOS versus 8 MB on Linux, so a default std::thread overflows its stack-guard page and + // aborts with SIGBUS on macOS arm64. The libuv pool that AsyncOperation runs on sizes its + // threads from RLIMIT_STACK, which is why that path never hit this. Pin a generous stack so + // the worker has the same headroom on every platform. + static constexpr size_t WORKER_STACK_SIZE = 32UL * 1024 * 1024; + + void Queue() + { + auto self = shared_from_this(); + launch_detached_with_large_stack([self]() { + try { + self->_fn(self->_result); + self->_success = true; + } catch (const std::exception& e) { + self->_error = e.what(); + self->_success = false; + } catch (...) { + self->_error = "Unknown exception occurred during threaded async operation"; + self->_success = false; + } + + // Post completion to the JS main thread. The callback captures `self` + // (shared_ptr) so the object stays alive until the callback runs. + // napi_tsfn_blocking only blocks on queue insertion, not on callback + // completion, so we cannot use raw pointers here. + self->_completion_tsfn.BlockingCall([self](Napi::Env env, Napi::Function /*js_callback*/) { + if (self->_success) { + auto buf = Napi::Buffer::Copy(env, self->_result.data(), self->_result.size()); + self->_deferred->Resolve(buf); + } else { + auto error = Napi::Error::New(env, self->_error); + self->_deferred->Reject(error.Value()); + } + self->_completion_tsfn.Release(); + }); + }); + } + + // Launch `work` on a detached OS thread with an explicitly large stack (see WORKER_STACK_SIZE). + // std::thread cannot set a stack size, so use pthreads where available and fall back to a + // default-stack std::thread only if pthread creation is unavailable or fails. + static void launch_detached_with_large_stack(std::function work) + { +#ifndef _WIN32 + pthread_attr_t attr; + if (pthread_attr_init(&attr) == 0) { + pthread_attr_setstacksize(&attr, WORKER_STACK_SIZE); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + + auto* heap_work = new std::function(std::move(work)); + pthread_t tid; + int rc = pthread_create( + &tid, + &attr, + [](void* arg) -> void* { + std::unique_ptr> fn(static_cast*>(arg)); + (*fn)(); + return nullptr; + }, + heap_work); + pthread_attr_destroy(&attr); + + if (rc == 0) { + return; + } + + // pthread_create failed; reclaim the work and fall back to a default std::thread. + std::unique_ptr> reclaimed(heap_work); + work = std::move(*reclaimed); + } +#endif + std::thread(std::move(work)).detach(); + } + + async_fn _fn; + std::shared_ptr _deferred; + Napi::ThreadSafeFunction _completion_tsfn; + msgpack::sbuffer _result; + bool _success = false; + std::string _error; +}; + +} // namespace bb::nodejs diff --git a/native-packages/kvdb/cpp/src/util/message_processor.hpp b/native-packages/kvdb/cpp/src/util/message_processor.hpp new file mode 100644 index 000000000000..06d4762e6826 --- /dev/null +++ b/native-packages/kvdb/cpp/src/util/message_processor.hpp @@ -0,0 +1,128 @@ +#pragma once + +#include "messaging/dispatcher.hpp" +#include "messaging/header.hpp" +#include "util/async_op.hpp" +#include "napi.h" +#include + +namespace bb::nodejs { + +class AsyncMessageProcessor { + public: + template + void register_handler(uint32_t msgType, T* self, R (T::*handler)() const, bool unique = false) + { + register_handler(msgType, self, handler, unique); + } + + template + void register_handler(uint32_t msgType, T* self, R (T::*handler)(), bool unique = false) + { + _register_handler( + msgType, [=](auto, const msgpack::object&) { return (self->*handler)(); }, unique); + } + + template + void register_handler(uint32_t msgType, T* self, R (T::*handler)(const P&) const, bool unique = false) + { + register_handler(msgType, self, handler, unique); + } + + template + void register_handler(uint32_t msgType, T* self, R (T::*handler)(const P&), bool unique = false) + { + _register_handler, R>( + msgType, + [=](const messaging::TypedMessage

& req, const msgpack::object&) { return (self->*handler)(req.value); }, + unique); + } + + template + void register_handler(uint32_t msgType, + T* self, + R (T::*handler)(const P&, const msgpack::object&) const, + bool unique = false) + { + register_handler(msgType, self, handler, unique); + } + + template + void register_handler(uint32_t msgType, + T* self, + R (T::*handler)(const P&, const msgpack::object&), + bool unique = false) + { + _register_handler, R>( + msgType, + [=](const messaging::TypedMessage

& req, const msgpack::object& obj) { + return (self->*handler)(req.value, obj); + }, + unique); + } + + Napi::Promise process_message(const Napi::CallbackInfo& info) + { + Napi::Env env = info.Env(); + // keep this in a shared pointer so that AsyncOperation can resolve/reject the promise once the execution is + // complete on an separate thread + auto deferred = std::make_shared(env); + + if (!open) { + deferred->Reject(Napi::TypeError::New(env, "Message processor is closed").Value()); + } else if (info.Length() < 1) { + deferred->Reject(Napi::TypeError::New(env, "Wrong number of arguments").Value()); + } else if (!info[0].IsBuffer()) { + deferred->Reject(Napi::TypeError::New(env, "Argument must be a buffer").Value()); + } else { + auto buffer = info[0].As>(); + size_t length = buffer.Length(); + // we mustn't access the Napi::Env outside of this top-level function + // so copy the data to a variable we own + // and make it a shared pointer so that it doesn't get destroyed as soon as we exit this code block + auto data = std::make_shared>(length); + std::copy_n(buffer.Data(), length, data->data()); + + auto* op = new bb::nodejs::AsyncOperation(env, deferred, [data, this, length](msgpack::sbuffer& buf) { + msgpack::object_handle obj_handle = msgpack::unpack(data->data(), length); + msgpack::object obj = obj_handle.get(); + dispatcher.on_new_data(obj, buf); + }); + + // Napi is now responsible for destroying this object + op->Queue(); + } + + return deferred->Promise(); + } + + void close() { open = false; } + + private: + bb::messaging::MessageDispatcher dispatcher; + std::atomic_bool open = true; + + template + void _register_handler(uint32_t msgType, + const std::function& fn, + bool unique = false) + { + dispatcher.register_target( + msgType, + [=](msgpack::object& obj, msgpack::sbuffer& buffer) { + P req_msg; + obj.convert(req_msg); + + R response = fn(req_msg, obj); + + bb::messaging::MsgHeader header(req_msg.header.messageId); + bb::messaging::TypedMessage resp_msg(msgType, header, response); + msgpack::pack(buffer, resp_msg); + + return true; + }, + unique); + } +}; + +} // namespace bb::nodejs diff --git a/native-packages/kvdb/cpp/src/util/promise.cpp b/native-packages/kvdb/cpp/src/util/promise.cpp new file mode 100644 index 000000000000..9216566c222c --- /dev/null +++ b/native-packages/kvdb/cpp/src/util/promise.cpp @@ -0,0 +1,12 @@ + +#include "napi.h" + +namespace bb::nodejs { + +Napi::Promise promise_reject(const Napi::Env& env, const Napi::Value& err) +{ + auto def = Napi::Promise::Deferred::New(env); + def.Reject(err); + return def.Promise(); +} +} // namespace bb::nodejs diff --git a/native-packages/kvdb/cpp/src/util/promise.hpp b/native-packages/kvdb/cpp/src/util/promise.hpp new file mode 100644 index 000000000000..4160d92eedb4 --- /dev/null +++ b/native-packages/kvdb/cpp/src/util/promise.hpp @@ -0,0 +1,9 @@ + +#pragma once + +#include "messaging/dispatcher.hpp" +#include "napi.h" + +namespace bb::nodejs { +Napi::Promise promise_reject(const Napi::Env& env, const Napi::Value& err); +} diff --git a/native-packages/kvdb/cpp/yarn.lock b/native-packages/kvdb/cpp/yarn.lock new file mode 100644 index 000000000000..ed3111738283 --- /dev/null +++ b/native-packages/kvdb/cpp/yarn.lock @@ -0,0 +1,579 @@ +# This file is generated by running "yarn install" inside your project. +# Manual changes might be lost - proceed with caution! + +__metadata: + version: 8 + cacheKey: 10c0 + +"@isaacs/fs-minipass@npm:^4.0.0": + version: 4.0.1 + resolution: "@isaacs/fs-minipass@npm:4.0.1" + dependencies: + minipass: "npm:^7.0.4" + checksum: 10c0/c25b6dc1598790d5b55c0947a9b7d111cfa92594db5296c3b907e2f533c033666f692a3939eadac17b1c7c40d362d0b0635dc874cbfe3e70db7c2b07cc97a5d2 + languageName: node + linkType: hard + +"@npmcli/agent@npm:^4.0.0": + version: 4.0.0 + resolution: "@npmcli/agent@npm:4.0.0" + dependencies: + agent-base: "npm:^7.1.0" + http-proxy-agent: "npm:^7.0.0" + https-proxy-agent: "npm:^7.0.1" + lru-cache: "npm:^11.2.1" + socks-proxy-agent: "npm:^8.0.3" + checksum: 10c0/f7b5ce0f3dd42c3f8c6546e8433573d8049f67ef11ec22aa4704bc41483122f68bf97752e06302c455ead667af5cb753e6a09bff06632bc465c1cfd4c4b75a53 + languageName: node + linkType: hard + +"@npmcli/fs@npm:^5.0.0": + version: 5.0.0 + resolution: "@npmcli/fs@npm:5.0.0" + dependencies: + semver: "npm:^7.3.5" + checksum: 10c0/26e376d780f60ff16e874a0ac9bc3399186846baae0b6e1352286385ac134d900cc5dafaded77f38d77f86898fc923ae1cee9d7399f0275b1aa24878915d722b + languageName: node + linkType: hard + +"abbrev@npm:^4.0.0": + version: 4.0.0 + resolution: "abbrev@npm:4.0.0" + checksum: 10c0/b4cc16935235e80702fc90192e349e32f8ef0ed151ef506aa78c81a7c455ec18375c4125414b99f84b2e055199d66383e787675f0bcd87da7a4dbd59f9eac1d5 + languageName: node + linkType: hard + +"agent-base@npm:^7.1.0, agent-base@npm:^7.1.2": + version: 7.1.4 + resolution: "agent-base@npm:7.1.4" + checksum: 10c0/c2c9ab7599692d594b6a161559ada307b7a624fa4c7b03e3afdb5a5e31cd0e53269115b620fcab024c5ac6a6f37fa5eb2e004f076ad30f5f7e6b8b671f7b35fe + languageName: node + linkType: hard + +"balanced-match@npm:^4.0.2": + version: 4.0.4 + resolution: "balanced-match@npm:4.0.4" + checksum: 10c0/07e86102a3eb2ee2a6a1a89164f29d0dbaebd28f2ca3f5ca786f36b8b23d9e417eb3be45a4acf754f837be5ac0a2317de90d3fcb7f4f4dc95720a1f36b26a17b + languageName: node + linkType: hard + +"brace-expansion@npm:^5.0.2": + version: 5.0.4 + resolution: "brace-expansion@npm:5.0.4" + dependencies: + balanced-match: "npm:^4.0.2" + checksum: 10c0/359cbcfa80b2eb914ca1f3440e92313fbfe7919ee6b274c35db55bec555aded69dac5ee78f102cec90c35f98c20fa43d10936d0cd9978158823c249257e1643a + languageName: node + linkType: hard + +"cacache@npm:^20.0.1": + version: 20.0.3 + resolution: "cacache@npm:20.0.3" + dependencies: + "@npmcli/fs": "npm:^5.0.0" + fs-minipass: "npm:^3.0.0" + glob: "npm:^13.0.0" + lru-cache: "npm:^11.1.0" + minipass: "npm:^7.0.3" + minipass-collect: "npm:^2.0.1" + minipass-flush: "npm:^1.0.5" + minipass-pipeline: "npm:^1.2.4" + p-map: "npm:^7.0.2" + ssri: "npm:^13.0.0" + unique-filename: "npm:^5.0.0" + checksum: 10c0/c7da1ca694d20e8f8aedabd21dc11518f809a7d2b59aa76a1fc655db5a9e62379e465c157ddd2afe34b19230808882288effa6911b2de26a088a6d5645123462 + languageName: node + linkType: hard + +"chownr@npm:^3.0.0": + version: 3.0.0 + resolution: "chownr@npm:3.0.0" + checksum: 10c0/43925b87700f7e3893296c8e9c56cc58f926411cce3a6e5898136daaf08f08b9a8eb76d37d3267e707d0dcc17aed2e2ebdf5848c0c3ce95cf910a919935c1b10 + languageName: node + linkType: hard + +"debug@npm:4, debug@npm:^4.3.4": + version: 4.4.3 + resolution: "debug@npm:4.4.3" + dependencies: + ms: "npm:^2.1.3" + peerDependenciesMeta: + supports-color: + optional: true + checksum: 10c0/d79136ec6c83ecbefd0f6a5593da6a9c91ec4d7ddc4b54c883d6e71ec9accb5f67a1a5e96d00a328196b5b5c86d365e98d8a3a70856aaf16b4e7b1985e67f5a6 + languageName: node + linkType: hard + +"encoding@npm:^0.1.13": + version: 0.1.13 + resolution: "encoding@npm:0.1.13" + dependencies: + iconv-lite: "npm:^0.6.2" + checksum: 10c0/36d938712ff00fe1f4bac88b43bcffb5930c1efa57bbcdca9d67e1d9d6c57cfb1200fb01efe0f3109b2ce99b231f90779532814a81370a1bd3274a0f58585039 + languageName: node + linkType: hard + +"env-paths@npm:^2.2.0": + version: 2.2.1 + resolution: "env-paths@npm:2.2.1" + checksum: 10c0/285325677bf00e30845e330eec32894f5105529db97496ee3f598478e50f008c5352a41a30e5e72ec9de8a542b5a570b85699cd63bd2bc646dbcb9f311d83bc4 + languageName: node + linkType: hard + +"err-code@npm:^2.0.2": + version: 2.0.3 + resolution: "err-code@npm:2.0.3" + checksum: 10c0/b642f7b4dd4a376e954947550a3065a9ece6733ab8e51ad80db727aaae0817c2e99b02a97a3d6cecc648a97848305e728289cf312d09af395403a90c9d4d8a66 + languageName: node + linkType: hard + +"exponential-backoff@npm:^3.1.1": + version: 3.1.3 + resolution: "exponential-backoff@npm:3.1.3" + checksum: 10c0/77e3ae682b7b1f4972f563c6dbcd2b0d54ac679e62d5d32f3e5085feba20483cf28bd505543f520e287a56d4d55a28d7874299941faf637e779a1aa5994d1267 + languageName: node + linkType: hard + +"fdir@npm:^6.5.0": + version: 6.5.0 + resolution: "fdir@npm:6.5.0" + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + checksum: 10c0/e345083c4306b3aed6cb8ec551e26c36bab5c511e99ea4576a16750ddc8d3240e63826cc624f5ae17ad4dc82e68a253213b60d556c11bfad064b7607847ed07f + languageName: node + linkType: hard + +"fs-minipass@npm:^3.0.0": + version: 3.0.3 + resolution: "fs-minipass@npm:3.0.3" + dependencies: + minipass: "npm:^7.0.3" + checksum: 10c0/63e80da2ff9b621e2cb1596abcb9207f1cf82b968b116ccd7b959e3323144cce7fb141462200971c38bbf2ecca51695069db45265705bed09a7cd93ae5b89f94 + languageName: node + linkType: hard + +"glob@npm:^13.0.0": + version: 13.0.2 + resolution: "glob@npm:13.0.2" + dependencies: + minimatch: "npm:^10.1.2" + minipass: "npm:^7.1.2" + path-scurry: "npm:^2.0.0" + checksum: 10c0/3d4b09efa922c4cba9be6d5b9efae14384e7422aeb886eb35fba8a94820b8281474b8d3f16927127fb1a0c8580e18fc00e3fda03c8dc31fa0af3ba918edeeb04 + languageName: node + linkType: hard + +"graceful-fs@npm:^4.2.6": + version: 4.2.11 + resolution: "graceful-fs@npm:4.2.11" + checksum: 10c0/386d011a553e02bc594ac2ca0bd6d9e4c22d7fa8cfbfc448a6d148c59ea881b092db9dbe3547ae4b88e55f1b01f7c4a2ecc53b310c042793e63aa44cf6c257f2 + languageName: node + linkType: hard + +"http-cache-semantics@npm:^4.1.1": + version: 4.2.0 + resolution: "http-cache-semantics@npm:4.2.0" + checksum: 10c0/45b66a945cf13ec2d1f29432277201313babf4a01d9e52f44b31ca923434083afeca03f18417f599c9ab3d0e7b618ceb21257542338b57c54b710463b4a53e37 + languageName: node + linkType: hard + +"http-proxy-agent@npm:^7.0.0": + version: 7.0.2 + resolution: "http-proxy-agent@npm:7.0.2" + dependencies: + agent-base: "npm:^7.1.0" + debug: "npm:^4.3.4" + checksum: 10c0/4207b06a4580fb85dd6dff521f0abf6db517489e70863dca1a0291daa7f2d3d2d6015a57bd702af068ea5cf9f1f6ff72314f5f5b4228d299c0904135d2aef921 + languageName: node + linkType: hard + +"https-proxy-agent@npm:^7.0.1": + version: 7.0.6 + resolution: "https-proxy-agent@npm:7.0.6" + dependencies: + agent-base: "npm:^7.1.2" + debug: "npm:4" + checksum: 10c0/f729219bc735edb621fa30e6e84e60ee5d00802b8247aac0d7b79b0bd6d4b3294737a337b93b86a0bd9e68099d031858a39260c976dc14cdbba238ba1f8779ac + languageName: node + linkType: hard + +"iconv-lite@npm:^0.6.2": + version: 0.6.3 + resolution: "iconv-lite@npm:0.6.3" + dependencies: + safer-buffer: "npm:>= 2.1.2 < 3.0.0" + checksum: 10c0/98102bc66b33fcf5ac044099d1257ba0b7ad5e3ccd3221f34dd508ab4070edff183276221684e1e0555b145fce0850c9f7d2b60a9fcac50fbb4ea0d6e845a3b1 + languageName: node + linkType: hard + +"imurmurhash@npm:^0.1.4": + version: 0.1.4 + resolution: "imurmurhash@npm:0.1.4" + checksum: 10c0/8b51313850dd33605c6c9d3fd9638b714f4c4c40250cff658209f30d40da60f78992fb2df5dabee4acf589a6a82bbc79ad5486550754bd9ec4e3fc0d4a57d6a6 + languageName: node + linkType: hard + +"ip-address@npm:^10.0.1": + version: 10.1.0 + resolution: "ip-address@npm:10.1.0" + checksum: 10c0/0103516cfa93f6433b3bd7333fa876eb21263912329bfa47010af5e16934eeeff86f3d2ae700a3744a137839ddfad62b900c7a445607884a49b5d1e32a3d7566 + languageName: node + linkType: hard + +"isexe@npm:^4.0.0": + version: 4.0.0 + resolution: "isexe@npm:4.0.0" + checksum: 10c0/5884815115bceac452877659a9c7726382531592f43dc29e5d48b7c4100661aed54018cb90bd36cb2eaeba521092570769167acbb95c18d39afdccbcca06c5ce + languageName: node + linkType: hard + +"lru-cache@npm:^11.0.0, lru-cache@npm:^11.1.0, lru-cache@npm:^11.2.1": + version: 11.2.6 + resolution: "lru-cache@npm:11.2.6" + checksum: 10c0/73bbffb298760e71b2bfe8ebc16a311c6a60ceddbba919cfedfd8635c2d125fbfb5a39b71818200e67973b11f8d59c5a9e31d6f90722e340e90393663a66e5cd + languageName: node + linkType: hard + +"make-fetch-happen@npm:^15.0.0": + version: 15.0.3 + resolution: "make-fetch-happen@npm:15.0.3" + dependencies: + "@npmcli/agent": "npm:^4.0.0" + cacache: "npm:^20.0.1" + http-cache-semantics: "npm:^4.1.1" + minipass: "npm:^7.0.2" + minipass-fetch: "npm:^5.0.0" + minipass-flush: "npm:^1.0.5" + minipass-pipeline: "npm:^1.2.4" + negotiator: "npm:^1.0.0" + proc-log: "npm:^6.0.0" + promise-retry: "npm:^2.0.1" + ssri: "npm:^13.0.0" + checksum: 10c0/525f74915660be60b616bcbd267c4a5b59481b073ba125e45c9c3a041bb1a47a2bd0ae79d028eb6f5f95bf9851a4158423f5068539c3093621abb64027e8e461 + languageName: node + linkType: hard + +"minimatch@npm:^10.1.2": + version: 10.2.4 + resolution: "minimatch@npm:10.2.4" + dependencies: + brace-expansion: "npm:^5.0.2" + checksum: 10c0/35f3dfb7b99b51efd46afd378486889f590e7efb10e0f6a10ba6800428cf65c9a8dedb74427d0570b318d749b543dc4e85f06d46d2858bc8cac7e1eb49a95945 + languageName: node + linkType: hard + +"minipass-collect@npm:^2.0.1": + version: 2.0.1 + resolution: "minipass-collect@npm:2.0.1" + dependencies: + minipass: "npm:^7.0.3" + checksum: 10c0/5167e73f62bb74cc5019594709c77e6a742051a647fe9499abf03c71dca75515b7959d67a764bdc4f8b361cf897fbf25e2d9869ee039203ed45240f48b9aa06e + languageName: node + linkType: hard + +"minipass-fetch@npm:^5.0.0": + version: 5.0.1 + resolution: "minipass-fetch@npm:5.0.1" + dependencies: + encoding: "npm:^0.1.13" + minipass: "npm:^7.0.3" + minipass-sized: "npm:^2.0.0" + minizlib: "npm:^3.0.1" + dependenciesMeta: + encoding: + optional: true + checksum: 10c0/50bcf48c9841ebb25e29a2817468595219c72cfffc7c175a1d7327843c8bef9b72cb01778f46df7eca695dfe47ab98e6167af4cb026ddd80f660842919a5193c + languageName: node + linkType: hard + +"minipass-flush@npm:^1.0.5": + version: 1.0.5 + resolution: "minipass-flush@npm:1.0.5" + dependencies: + minipass: "npm:^3.0.0" + checksum: 10c0/2a51b63feb799d2bb34669205eee7c0eaf9dce01883261a5b77410c9408aa447e478efd191b4de6fc1101e796ff5892f8443ef20d9544385819093dbb32d36bd + languageName: node + linkType: hard + +"minipass-pipeline@npm:^1.2.4": + version: 1.2.4 + resolution: "minipass-pipeline@npm:1.2.4" + dependencies: + minipass: "npm:^3.0.0" + checksum: 10c0/cbda57cea20b140b797505dc2cac71581a70b3247b84480c1fed5ca5ba46c25ecc25f68bfc9e6dcb1a6e9017dab5c7ada5eab73ad4f0a49d84e35093e0c643f2 + languageName: node + linkType: hard + +"minipass-sized@npm:^2.0.0": + version: 2.0.0 + resolution: "minipass-sized@npm:2.0.0" + dependencies: + minipass: "npm:^7.1.2" + checksum: 10c0/f9201696a6f6d68610d04c9c83e3d2e5cb9c026aae1c8cbf7e17f386105cb79c1bb088dbc21bf0b1eb4f3fb5df384fd1e7aa3bf1f33868c416ae8c8a92679db8 + languageName: node + linkType: hard + +"minipass@npm:^3.0.0": + version: 3.3.6 + resolution: "minipass@npm:3.3.6" + dependencies: + yallist: "npm:^4.0.0" + checksum: 10c0/a114746943afa1dbbca8249e706d1d38b85ed1298b530f5808ce51f8e9e941962e2a5ad2e00eae7dd21d8a4aae6586a66d4216d1a259385e9d0358f0c1eba16c + languageName: node + linkType: hard + +"minipass@npm:^7.0.2, minipass@npm:^7.0.3, minipass@npm:^7.0.4, minipass@npm:^7.1.2": + version: 7.1.2 + resolution: "minipass@npm:7.1.2" + checksum: 10c0/b0fd20bb9fb56e5fa9a8bfac539e8915ae07430a619e4b86ff71f5fc757ef3924b23b2c4230393af1eda647ed3d75739e4e0acb250a6b1eb277cf7f8fe449557 + languageName: node + linkType: hard + +"minizlib@npm:^3.0.1, minizlib@npm:^3.1.0": + version: 3.1.0 + resolution: "minizlib@npm:3.1.0" + dependencies: + minipass: "npm:^7.1.2" + checksum: 10c0/5aad75ab0090b8266069c9aabe582c021ae53eb33c6c691054a13a45db3b4f91a7fb1bd79151e6b4e9e9a86727b522527c0a06ec7d45206b745d54cd3097bcec + languageName: node + linkType: hard + +"ms@npm:^2.1.3": + version: 2.1.3 + resolution: "ms@npm:2.1.3" + checksum: 10c0/d924b57e7312b3b63ad21fc5b3dc0af5e78d61a1fc7cfb5457edaf26326bf62be5307cc87ffb6862ef1c2b33b0233cdb5d4f01c4c958cc0d660948b65a287a48 + languageName: node + linkType: hard + +"negotiator@npm:^1.0.0": + version: 1.0.0 + resolution: "negotiator@npm:1.0.0" + checksum: 10c0/4c559dd52669ea48e1914f9d634227c561221dd54734070791f999c52ed0ff36e437b2e07d5c1f6e32909fc625fe46491c16e4a8f0572567d4dd15c3a4fda04b + languageName: node + linkType: hard + +"node-addon-api@npm:^8.0.0": + version: 8.0.0 + resolution: "node-addon-api@npm:8.0.0" + dependencies: + node-gyp: "npm:latest" + checksum: 10c0/20eb231362cc07c62d9839164473744d985be5d82685214f3750d990d9f61ef366e0ba112a766c925d640ed29b2a500b83568e895dc2444dcd5db01e615aac2b + languageName: node + linkType: hard + +"node-api-headers@npm:^1.1.0": + version: 1.1.0 + resolution: "node-api-headers@npm:1.1.0" + checksum: 10c0/7806d71077348ea199034e8c90a9147038d37fcccc1b85717e48c095fe31783a4f909f5daced4506e6cbce93fba91220bb3fc8626ee0640d26de9860f6500174 + languageName: node + linkType: hard + +"node-gyp@npm:latest": + version: 12.2.0 + resolution: "node-gyp@npm:12.2.0" + dependencies: + env-paths: "npm:^2.2.0" + exponential-backoff: "npm:^3.1.1" + graceful-fs: "npm:^4.2.6" + make-fetch-happen: "npm:^15.0.0" + nopt: "npm:^9.0.0" + proc-log: "npm:^6.0.0" + semver: "npm:^7.3.5" + tar: "npm:^7.5.4" + tinyglobby: "npm:^0.2.12" + which: "npm:^6.0.0" + bin: + node-gyp: bin/node-gyp.js + checksum: 10c0/3ed046746a5a7d90950cd8b0547332b06598443f31fe213ef4332a7174c7b7d259e1704835feda79b87d3f02e59d7791842aac60642ede4396ab25fdf0f8f759 + languageName: node + linkType: hard + +"nodejs_module@workspace:.": + version: 0.0.0-use.local + resolution: "nodejs_module@workspace:." + dependencies: + node-addon-api: "npm:^8.0.0" + node-api-headers: "npm:^1.1.0" + languageName: unknown + linkType: soft + +"nopt@npm:^9.0.0": + version: 9.0.0 + resolution: "nopt@npm:9.0.0" + dependencies: + abbrev: "npm:^4.0.0" + bin: + nopt: bin/nopt.js + checksum: 10c0/1822eb6f9b020ef6f7a7516d7b64a8036e09666ea55ac40416c36e4b2b343122c3cff0e2f085675f53de1d2db99a2a89a60ccea1d120bcd6a5347bf6ceb4a7fd + languageName: node + linkType: hard + +"p-map@npm:^7.0.2": + version: 7.0.4 + resolution: "p-map@npm:7.0.4" + checksum: 10c0/a5030935d3cb2919d7e89454d1ce82141e6f9955413658b8c9403cfe379283770ed3048146b44cde168aa9e8c716505f196d5689db0ae3ce9a71521a2fef3abd + languageName: node + linkType: hard + +"path-scurry@npm:^2.0.0": + version: 2.0.1 + resolution: "path-scurry@npm:2.0.1" + dependencies: + lru-cache: "npm:^11.0.0" + minipass: "npm:^7.1.2" + checksum: 10c0/2a16ed0e81fbc43513e245aa5763354e25e787dab0d539581a6c3f0f967461a159ed6236b2559de23aa5b88e7dc32b469b6c47568833dd142a4b24b4f5cd2620 + languageName: node + linkType: hard + +"picomatch@npm:^4.0.3": + version: 4.0.3 + resolution: "picomatch@npm:4.0.3" + checksum: 10c0/9582c951e95eebee5434f59e426cddd228a7b97a0161a375aed4be244bd3fe8e3a31b846808ea14ef2c8a2527a6eeab7b3946a67d5979e81694654f939473ae2 + languageName: node + linkType: hard + +"proc-log@npm:^6.0.0": + version: 6.1.0 + resolution: "proc-log@npm:6.1.0" + checksum: 10c0/4f178d4062733ead9d71a9b1ab24ebcecdfe2250916a5b1555f04fe2eda972a0ec76fbaa8df1ad9c02707add6749219d118a4fc46dc56bdfe4dde4b47d80bb82 + languageName: node + linkType: hard + +"promise-retry@npm:^2.0.1": + version: 2.0.1 + resolution: "promise-retry@npm:2.0.1" + dependencies: + err-code: "npm:^2.0.2" + retry: "npm:^0.12.0" + checksum: 10c0/9c7045a1a2928094b5b9b15336dcd2a7b1c052f674550df63cc3f36cd44028e5080448175b6f6ca32b642de81150f5e7b1a98b728f15cb069f2dd60ac2616b96 + languageName: node + linkType: hard + +"retry@npm:^0.12.0": + version: 0.12.0 + resolution: "retry@npm:0.12.0" + checksum: 10c0/59933e8501727ba13ad73ef4a04d5280b3717fd650408460c987392efe9d7be2040778ed8ebe933c5cbd63da3dcc37919c141ef8af0a54a6e4fca5a2af177bfe + languageName: node + linkType: hard + +"safer-buffer@npm:>= 2.1.2 < 3.0.0": + version: 2.1.2 + resolution: "safer-buffer@npm:2.1.2" + checksum: 10c0/7e3c8b2e88a1841c9671094bbaeebd94448111dd90a81a1f606f3f67708a6ec57763b3b47f06da09fc6054193e0e6709e77325415dc8422b04497a8070fa02d4 + languageName: node + linkType: hard + +"semver@npm:^7.3.5": + version: 7.7.4 + resolution: "semver@npm:7.7.4" + bin: + semver: bin/semver.js + checksum: 10c0/5215ad0234e2845d4ea5bb9d836d42b03499546ddafb12075566899fc617f68794bb6f146076b6881d755de17d6c6cc73372555879ec7dce2c2feee947866ad2 + languageName: node + linkType: hard + +"smart-buffer@npm:^4.2.0": + version: 4.2.0 + resolution: "smart-buffer@npm:4.2.0" + checksum: 10c0/a16775323e1404dd43fabafe7460be13a471e021637bc7889468eb45ce6a6b207261f454e4e530a19500cc962c4cc5348583520843b363f4193cee5c00e1e539 + languageName: node + linkType: hard + +"socks-proxy-agent@npm:^8.0.3": + version: 8.0.5 + resolution: "socks-proxy-agent@npm:8.0.5" + dependencies: + agent-base: "npm:^7.1.2" + debug: "npm:^4.3.4" + socks: "npm:^2.8.3" + checksum: 10c0/5d2c6cecba6821389aabf18728325730504bf9bb1d9e342e7987a5d13badd7a98838cc9a55b8ed3cb866ad37cc23e1086f09c4d72d93105ce9dfe76330e9d2a6 + languageName: node + linkType: hard + +"socks@npm:^2.8.3": + version: 2.8.7 + resolution: "socks@npm:2.8.7" + dependencies: + ip-address: "npm:^10.0.1" + smart-buffer: "npm:^4.2.0" + checksum: 10c0/2805a43a1c4bcf9ebf6e018268d87b32b32b06fbbc1f9282573583acc155860dc361500f89c73bfbb157caa1b4ac78059eac0ef15d1811eb0ca75e0bdadbc9d2 + languageName: node + linkType: hard + +"ssri@npm:^13.0.0": + version: 13.0.1 + resolution: "ssri@npm:13.0.1" + dependencies: + minipass: "npm:^7.0.3" + checksum: 10c0/cf6408a18676c57ff2ed06b8a20dc64bb3e748e5c7e095332e6aecaa2b8422b1e94a739a8453bf65156a8a47afe23757ba4ab52d3ea3b62322dc40875763e17a + languageName: node + linkType: hard + +"tar@npm:^7.5.4": + version: 7.5.11 + resolution: "tar@npm:7.5.11" + dependencies: + "@isaacs/fs-minipass": "npm:^4.0.0" + chownr: "npm:^3.0.0" + minipass: "npm:^7.1.2" + minizlib: "npm:^3.1.0" + yallist: "npm:^5.0.0" + checksum: 10c0/b6bb420550ef50ef23356018155e956cd83282c97b6128d8d5cfe5740c57582d806a244b2ef0bf686a74ce526babe8b8b9061527623e935e850008d86d838929 + languageName: node + linkType: hard + +"tinyglobby@npm:^0.2.12": + version: 0.2.15 + resolution: "tinyglobby@npm:0.2.15" + dependencies: + fdir: "npm:^6.5.0" + picomatch: "npm:^4.0.3" + checksum: 10c0/869c31490d0d88eedb8305d178d4c75e7463e820df5a9b9d388291daf93e8b1eb5de1dad1c1e139767e4269fe75f3b10d5009b2cc14db96ff98986920a186844 + languageName: node + linkType: hard + +"unique-filename@npm:^5.0.0": + version: 5.0.0 + resolution: "unique-filename@npm:5.0.0" + dependencies: + unique-slug: "npm:^6.0.0" + checksum: 10c0/afb897e9cf4c2fb622ea716f7c2bb462001928fc5f437972213afdf1cc32101a230c0f1e9d96fc91ee5185eca0f2feb34127145874975f347be52eb91d6ccc2c + languageName: node + linkType: hard + +"unique-slug@npm:^6.0.0": + version: 6.0.0 + resolution: "unique-slug@npm:6.0.0" + dependencies: + imurmurhash: "npm:^0.1.4" + checksum: 10c0/da7ade4cb04eb33ad0499861f82fe95ce9c7c878b7139dc54d140ecfb6a6541c18a5c8dac16188b8b379fe62c0c1f1b710814baac910cde5f4fec06212126c6a + languageName: node + linkType: hard + +"which@npm:^6.0.0": + version: 6.0.1 + resolution: "which@npm:6.0.1" + dependencies: + isexe: "npm:^4.0.0" + bin: + node-which: bin/which.js + checksum: 10c0/7e710e54ea36d2d6183bee2f9caa27a3b47b9baf8dee55a199b736fcf85eab3b9df7556fca3d02b50af7f3dfba5ea3a45644189836df06267df457e354da66d5 + languageName: node + linkType: hard + +"yallist@npm:^4.0.0": + version: 4.0.0 + resolution: "yallist@npm:4.0.0" + checksum: 10c0/2286b5e8dbfe22204ab66e2ef5cc9bbb1e55dfc873bbe0d568aa943eb255d131890dfd5bf243637273d31119b870f49c18fcde2c6ffbb7a7a092b870dc90625a + languageName: node + linkType: hard + +"yallist@npm:^5.0.0": + version: 5.0.0 + resolution: "yallist@npm:5.0.0" + checksum: 10c0/a499c81ce6d4a1d260d4ea0f6d49ab4da09681e32c3f0472dee16667ed69d01dae63a3b81745a24bd78476ec4fcf856114cb4896ace738e01da34b2c42235416 + languageName: node + linkType: hard diff --git a/native-packages/kvdb/package.json b/native-packages/kvdb/package.json new file mode 100644 index 000000000000..6dabbfc25130 --- /dev/null +++ b/native-packages/kvdb/package.json @@ -0,0 +1,13 @@ +{ + "name": "@aztec/kvdb-packages", + "packageManager": "yarn@4.13.0", + "private": true, + "scripts": { + "build": "yarn workspace @aztec/kvdb build", + "clean": "yarn workspaces foreach -A -p -v run clean" + }, + "workspaces": [ + "ts", + "ts/packages/*" + ] +} diff --git a/native-packages/kvdb/ts/package.json b/native-packages/kvdb/ts/package.json new file mode 100644 index 000000000000..f92a5163ec1d --- /dev/null +++ b/native-packages/kvdb/ts/package.json @@ -0,0 +1,33 @@ +{ + "name": "@aztec/kvdb", + "version": "0.1.0", + "type": "module", + "exports": { + ".": { + "types": "./dest/index.d.ts", + "default": "./dest/index.js" + } + }, + "files": [ + "dest/", + "README.md" + ], + "scripts": { + "clean": "rm -rf dest .tsbuildinfo", + "build": "tsc -p tsconfig.json", + "prepare_arch_packages": "./scripts/prepare_arch_packages.sh" + }, + "dependencies": { + "tslib": "^2.4.0" + }, + "optionalDependencies": { + "@aztec/kvdb-darwin-arm64": "0.1.0", + "@aztec/kvdb-darwin-x64": "0.1.0", + "@aztec/kvdb-linux-arm64": "0.1.0", + "@aztec/kvdb-linux-x64": "0.1.0" + }, + "devDependencies": { + "@types/node": "^22.15.17", + "typescript": "^5.3.3" + } +} diff --git a/native-packages/kvdb/ts/scripts/prepare_arch_packages.sh b/native-packages/kvdb/ts/scripts/prepare_arch_packages.sh new file mode 100755 index 000000000000..701b6f3e0f8e --- /dev/null +++ b/native-packages/kvdb/ts/scripts/prepare_arch_packages.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +declare -A PLATFORMS=( + ["amd64-linux"]="linux-x64 linux x64" + ["arm64-linux"]="linux-arm64 linux arm64" + ["amd64-macos"]="darwin-x64 darwin x64" + ["arm64-macos"]="darwin-arm64 darwin arm64" +) + +NAPI_BINARY="nodejs_module.node" +version=$(node -p "require('./package.json').version") + +declare -A BINARIES=() +for arg in "$@"; do + case "$arg" in + *=*) + key="${arg%%=*}" + value="${arg#*=}" + BINARIES["$key"]="$value" + ;; + *) + echo "Usage: npm run prepare_arch_packages -- [= ...]" >&2 + echo "Platforms: linux-x64, linux-arm64, darwin-x64, darwin-arm64" >&2 + exit 1 + ;; + esac +done + +for build_dir in "${!PLATFORMS[@]}"; do + read -r suffix os cpu <<< "${PLATFORMS[$build_dir]}" + pkg_name="@aztec/kvdb-${suffix}" + out_dir="packages/kvdb-${suffix}" + binary_path="${BINARIES[$suffix]:-${BINARIES[$build_dir]:-}}" + + if [ -z "$binary_path" ]; then + binary_path="build/${build_dir}/${NAPI_BINARY}" + fi + + # Always create the arch workspace stub so the workspace set (and lockfile) is + # stable on single-arch builds. The os/cpu fields make npm install only the + # matching package at runtime; the binary is copied in only where available. + rm -rf "${out_dir}" + mkdir -p "${out_dir}" + if [ -f "$binary_path" ]; then + cp "$binary_path" "${out_dir}/${NAPI_BINARY}" + else + echo "No binary for ${pkg_name} (${binary_path}); creating stub workspace package." + fi + + cat > "${out_dir}/package.json" < = { + "x86_64-linux": "@aztec/kvdb-linux-x64", + "x86_64-darwin": "@aztec/kvdb-darwin-x64", + "aarch64-linux": "@aztec/kvdb-linux-arm64", + "aarch64-darwin": "@aztec/kvdb-darwin-arm64", +}; + +const NAPI_BINARY = "nodejs_module.node"; + +function currentDir(): string { + return path.dirname(fileURLToPath(import.meta.url)); +} + +function detectPlatform(): Platform | null { + if (process.arch === "x64" && process.platform === "linux") + return "x86_64-linux"; + if (process.arch === "x64" && process.platform === "darwin") + return "x86_64-darwin"; + if (process.arch === "arm64" && process.platform === "linux") + return "aarch64-linux"; + if (process.arch === "arm64" && process.platform === "darwin") + return "aarch64-darwin"; + return null; +} + +function findArchPackageDir(platform: Platform): string | null { + const packageName = PLATFORM_TO_PACKAGE[platform]; + try { + const require = createRequire(import.meta.url); + return path.dirname(require.resolve(packageName + "/package.json")); + } catch { + const siblingPackageDir = path.join( + currentDir(), + "..", + "packages", + packageName.split("/").pop()!, + ); + return fs.existsSync(path.join(siblingPackageDir, "package.json")) + ? siblingPackageDir + : null; + } +} + +/** + * Locate the kvdb NAPI addon (nodejs_module.node) for the current platform. + * @param customPath Optional explicit path to the addon (overrides detection). + * @returns Absolute path to the addon, or null if not found. + */ +export function findNapiBinary(customPath?: string): string | null { + if (customPath) { + return fs.existsSync(customPath) ? path.resolve(customPath) : null; + } + + const envPath = process.env.AZTEC_KVDB_NAPI_PATH; + if (envPath) { + return fs.existsSync(envPath) ? path.resolve(envPath) : null; + } + + const platform = detectPlatform(); + if (!platform) { + return null; + } + + const archDir = findArchPackageDir(platform); + if (archDir) { + const candidate = path.join(archDir, NAPI_BINARY); + if (fs.existsSync(candidate)) { + return candidate; + } + } + + return null; +} diff --git a/native-packages/kvdb/ts/tsconfig.json b/native-packages/kvdb/ts/tsconfig.json new file mode 100644 index 000000000000..18049f35db9c --- /dev/null +++ b/native-packages/kvdb/ts/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "declaration": true, + "declarationMap": true, + "composite": true, + "outDir": "dest", + "rootDir": "src", + "tsBuildInfoFile": ".tsbuildinfo", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src/**/*.ts"] +} diff --git a/native-packages/kvdb/yarn.lock b/native-packages/kvdb/yarn.lock new file mode 100644 index 000000000000..23b6685c9d03 --- /dev/null +++ b/native-packages/kvdb/yarn.lock @@ -0,0 +1,102 @@ +# This file is generated by running "yarn install" inside your project. +# Manual changes might be lost - proceed with caution! + +__metadata: + version: 8 + cacheKey: 10c0 + +"@aztec/kvdb-darwin-arm64@npm:0.1.0, @aztec/kvdb-darwin-arm64@workspace:ts/packages/kvdb-darwin-arm64": + version: 0.0.0-use.local + resolution: "@aztec/kvdb-darwin-arm64@workspace:ts/packages/kvdb-darwin-arm64" + languageName: unknown + linkType: soft + +"@aztec/kvdb-darwin-x64@npm:0.1.0, @aztec/kvdb-darwin-x64@workspace:ts/packages/kvdb-darwin-x64": + version: 0.0.0-use.local + resolution: "@aztec/kvdb-darwin-x64@workspace:ts/packages/kvdb-darwin-x64" + languageName: unknown + linkType: soft + +"@aztec/kvdb-linux-arm64@npm:0.1.0, @aztec/kvdb-linux-arm64@workspace:ts/packages/kvdb-linux-arm64": + version: 0.0.0-use.local + resolution: "@aztec/kvdb-linux-arm64@workspace:ts/packages/kvdb-linux-arm64" + languageName: unknown + linkType: soft + +"@aztec/kvdb-linux-x64@npm:0.1.0, @aztec/kvdb-linux-x64@workspace:ts/packages/kvdb-linux-x64": + version: 0.0.0-use.local + resolution: "@aztec/kvdb-linux-x64@workspace:ts/packages/kvdb-linux-x64" + languageName: unknown + linkType: soft + +"@aztec/kvdb-packages@workspace:.": + version: 0.0.0-use.local + resolution: "@aztec/kvdb-packages@workspace:." + languageName: unknown + linkType: soft + +"@aztec/kvdb@workspace:ts": + version: 0.0.0-use.local + resolution: "@aztec/kvdb@workspace:ts" + dependencies: + "@aztec/kvdb-darwin-arm64": "npm:0.1.0" + "@aztec/kvdb-darwin-x64": "npm:0.1.0" + "@aztec/kvdb-linux-arm64": "npm:0.1.0" + "@aztec/kvdb-linux-x64": "npm:0.1.0" + "@types/node": "npm:^22.15.17" + tslib: "npm:^2.4.0" + typescript: "npm:^5.3.3" + dependenciesMeta: + "@aztec/kvdb-darwin-arm64": + optional: true + "@aztec/kvdb-darwin-x64": + optional: true + "@aztec/kvdb-linux-arm64": + optional: true + "@aztec/kvdb-linux-x64": + optional: true + languageName: unknown + linkType: soft + +"@types/node@npm:^22.15.17": + version: 22.20.0 + resolution: "@types/node@npm:22.20.0" + dependencies: + undici-types: "npm:~6.21.0" + checksum: 10c0/55d78223205bd5f81f043d71b7a5c8d8854b9ef44ef81291680943adb27fa5ba1f092658c87183d5bc8cf6baf6a57b81dad966eb3afa452cc301a615b6d9b20e + languageName: node + linkType: hard + +"tslib@npm:^2.4.0": + version: 2.8.1 + resolution: "tslib@npm:2.8.1" + checksum: 10c0/9c4759110a19c53f992d9aae23aac5ced636e99887b51b9e61def52611732872ff7668757d4e4c61f19691e36f4da981cd9485e869b4a7408d689f6bf1f14e62 + languageName: node + linkType: hard + +"typescript@npm:^5.3.3": + version: 5.9.3 + resolution: "typescript@npm:5.9.3" + bin: + tsc: bin/tsc + tsserver: bin/tsserver + checksum: 10c0/6bd7552ce39f97e711db5aa048f6f9995b53f1c52f7d8667c1abdc1700c68a76a308f579cd309ce6b53646deb4e9a1be7c813a93baaf0a28ccd536a30270e1c5 + languageName: node + linkType: hard + +"typescript@patch:typescript@npm%3A^5.3.3#optional!builtin": + version: 5.9.3 + resolution: "typescript@patch:typescript@npm%3A5.9.3#optional!builtin::version=5.9.3&hash=5786d5" + bin: + tsc: bin/tsc + tsserver: bin/tsserver + checksum: 10c0/ad09fdf7a756814dce65bc60c1657b40d44451346858eea230e10f2e95a289d9183b6e32e5c11e95acc0ccc214b4f36289dcad4bf1886b0adb84d711d336a430 + languageName: node + linkType: hard + +"undici-types@npm:~6.21.0": + version: 6.21.0 + resolution: "undici-types@npm:6.21.0" + checksum: 10c0/c01ed51829b10aa72fc3ce64b747f8e74ae9b60eafa19a7b46ef624403508a54c526ffab06a14a26b3120d055e1104d7abe7c9017e83ced038ea5cf52f8d5e04 + languageName: node + linkType: hard diff --git a/native-packages/lmdblib/.gitignore b/native-packages/lmdblib/.gitignore new file mode 100644 index 000000000000..568bb61b59d2 --- /dev/null +++ b/native-packages/lmdblib/.gitignore @@ -0,0 +1 @@ +cpp/build/ diff --git a/native-packages/lmdblib/.rebuild_patterns b/native-packages/lmdblib/.rebuild_patterns new file mode 100644 index 000000000000..9c628b1c31f5 --- /dev/null +++ b/native-packages/lmdblib/.rebuild_patterns @@ -0,0 +1,3 @@ +^native-packages/lmdblib/bootstrap\.sh$ +^native-packages/lmdblib/\.rebuild_patterns$ +^native-packages/lmdblib/cpp/ diff --git a/native-packages/lmdblib/bootstrap.sh b/native-packages/lmdblib/bootstrap.sh new file mode 100755 index 000000000000..b8d41523d981 --- /dev/null +++ b/native-packages/lmdblib/bootstrap.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +source $(git rev-parse --show-toplevel)/ci3/source_bootstrap + +ROOT=$(git rev-parse --show-toplevel) +PKG="$ROOT/native-packages/lmdblib" + +hash=$(hash_str \ + $(cache_content_hash .rebuild_patterns)) + +# Build liblmdblib.a + liblmdb.a. lmdblib is barretenberg-free: it builds against +# its own deps (lmdb, msgpack-c) only. Both archives are consumed by the kvdb NAPI +# addon and the wsdb service. +function build_native { + local build_dir="cpp/build" + CC=$(which clang) CXX=$(which clang++) cmake -S cpp -B "$build_dir" -G Ninja >/dev/null + cmake --build "$build_dir" --target lmdblib +} + +function build { + echo_header "lmdblib build" + build_native +} + +function clean { + rm -rf cpp/build +} + +export -f build_native build clean + +case "$cmd" in + "") + build + ;; + "hash") + echo "$hash" + ;; + *) + default_cmd_handler "$@" + ;; +esac diff --git a/native-packages/lmdblib/cpp/CMakeLists.txt b/native-packages/lmdblib/cpp/CMakeLists.txt new file mode 100644 index 000000000000..e175973efadd --- /dev/null +++ b/native-packages/lmdblib/cpp/CMakeLists.txt @@ -0,0 +1,101 @@ +# Standalone build for lmdblib — the LMDB C++ wrapper. +# +# This package owns the lmdb wrapper that barretenberg no longer needs (the +# in-memory merkle core is lmdb-free). It builds two static archives that the +# kvdb NAPI addon and the wsdb service link against: +# - liblmdb.a : the upstream LMDB C library (pinned to bb's commit) +# - liblmdblib.a : the bb::lmdblib C++ wrapper objects +# +# It depends on barretenberg only for *headers* (serialize, common, numeric, and +# crypto/merkle_tree/db_stats.hpp for the shared DBStats type). The wrapper's +# unresolved bb symbols are satisfied when a consumer links libbarretenberg.a. + +cmake_minimum_required(VERSION 3.24) +# C is enabled so CMAKE_C_COMPILER is populated for lmdb's ExternalProject build. +project(aztec_lmdblib CXX C) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) + +# Match barretenberg's prebuilt archive (Release -> -O3 -DNDEBUG): bb headers compiled +# here must use the same NDEBUG/opt as bb to avoid ODR-divergent inlined code. +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif() + +# --------------------------------------------------------------------------- +# Third-party deps (own copies; commits must match barretenberg/cpp/cmake/*). +# --------------------------------------------------------------------------- +set(DEPS_PREFIX "${CMAKE_BINARY_DIR}/_deps") +include(ExternalProject) + +# lmdb: build our own liblmdb.a (pinned to bb's commit in cmake/lmdb.cmake). +set(LMDB_PREFIX "${DEPS_PREFIX}/lmdb") +set(LMDB_INCLUDE "${LMDB_PREFIX}/src/lmdb_repo/libraries/liblmdb") +set(LMDB_LIB "${LMDB_INCLUDE}/liblmdb.a") +ExternalProject_Add( + lmdb_repo + PREFIX ${LMDB_PREFIX} + DOWNLOAD_COMMAND + sh -c "mkdir -p ${LMDB_PREFIX}/src/lmdb_repo && cd ${LMDB_PREFIX}/src/lmdb_repo && git init --quiet && (git remote add origin https://github.com/LMDB/lmdb.git 2>/dev/null || true) && git fetch --depth 1 origin --quiet ddd0a773e2f44d38e4e31ec9ed81af81f4e4ccbb && git reset --quiet --hard FETCH_HEAD" + SOURCE_DIR ${LMDB_PREFIX}/src/lmdb_repo + BUILD_IN_SOURCE YES + CONFIGURE_COMMAND "" + BUILD_COMMAND ${CMAKE_COMMAND} -E env --unset=CFLAGS --unset=CXXFLAGS CC=${CMAKE_C_COMPILER}${CMAKE_C_COMPILER_ARG1} AR=${CMAKE_AR} make -e -C libraries/liblmdb XCFLAGS=-fPIC liblmdb.a + INSTALL_COMMAND "" + UPDATE_COMMAND "" + BUILD_BYPRODUCTS ${LMDB_LIB} +) +add_library(lmdb STATIC IMPORTED GLOBAL) +add_dependencies(lmdb lmdb_repo) +set_target_properties(lmdb PROPERTIES IMPORTED_LOCATION ${LMDB_LIB}) + +# msgpack-c: header-only; fetch for includes (pinned to bb's commit in cmake/msgpack.cmake). +set(MSGPACK_PREFIX "${DEPS_PREFIX}/msgpack-c") +set(MSGPACK_INCLUDE "${MSGPACK_PREFIX}/src/msgpack-c/include") +ExternalProject_Add( + msgpack-c + PREFIX ${MSGPACK_PREFIX} + DOWNLOAD_COMMAND + sh -c "mkdir -p ${MSGPACK_PREFIX}/src/msgpack-c && cd ${MSGPACK_PREFIX}/src/msgpack-c && git init --quiet && (git remote add origin https://github.com/AztecProtocol/msgpack-c.git 2>/dev/null || true) && git fetch --depth 1 origin --quiet c0334576ed657fb3b3c49e8e61402989fb84146d && git reset --quiet --hard FETCH_HEAD" + SOURCE_DIR ${MSGPACK_PREFIX}/src/msgpack-c + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + UPDATE_COMMAND "" +) + +# lmdblib is barretenberg-free: it links no bb archives and includes no bb headers. +# It is still compiled with libc++ and -march=skylake so its objects are ABI- and +# arch-compatible when linked into consumers (wsdb, kvdb) alongside bb's archives. +add_compile_definitions(MSGPACK_NO_BOOST MSGPACK_USE_STD_VARIANT_ADAPTOR NO_PAR_ALGOS) +add_compile_options(-march=skylake -fconstexpr-steps=100000000 -fbracket-depth=1024) +add_compile_options(-Wno-deprecated-declarations -Wno-missing-field-initializers) +add_compile_options(-stdlib=libc++) + +include_directories( + ${CMAKE_CURRENT_SOURCE_DIR}/src + ${MSGPACK_INCLUDE} + ${LMDB_INCLUDE} +) + +# --------------------------------------------------------------------------- +# liblmdblib.a — the C++ wrapper objects. +# --------------------------------------------------------------------------- +set(LMDBLIB_SOURCES + src/lmdblib/lmdb_cursor.cpp + src/lmdblib/lmdb_database.cpp + src/lmdblib/lmdb_db_transaction.cpp + src/lmdblib/lmdb_environment.cpp + src/lmdblib/lmdb_helpers.cpp + src/lmdblib/lmdb_read_transaction.cpp + src/lmdblib/lmdb_store.cpp + src/lmdblib/lmdb_store_base.cpp + src/lmdblib/lmdb_transaction.cpp + src/lmdblib/lmdb_write_transaction.cpp + src/lmdblib/queries.cpp +) +add_library(lmdblib STATIC ${LMDBLIB_SOURCES}) +add_dependencies(lmdblib lmdb_repo msgpack-c) diff --git a/native-packages/lmdblib/cpp/src/lmdblib/fixtures.hpp b/native-packages/lmdblib/cpp/src/lmdblib/fixtures.hpp new file mode 100644 index 000000000000..74e9978fc480 --- /dev/null +++ b/native-packages/lmdblib/cpp/src/lmdblib/fixtures.hpp @@ -0,0 +1,36 @@ +#include "lmdblib/types.hpp" +#include +#include +#include + +namespace bb::lmdblib { +const uint32_t NUM_VALUES = 1024; + +inline std::string random_string() { + static thread_local std::mt19937 gen(std::random_device{}()); + std::stringstream ss; + ss << std::uniform_int_distribution{}(gen); + return ss.str(); +} + +inline std::string random_temp_directory() { + std::stringstream ss; + ss << "/tmp/lmdb/" << random_string(); + return ss.str(); +} + +inline std::vector serialise(std::string key) { + std::vector data(key.begin(), key.end()); + return data; +} + +inline Key get_key(int64_t keyCount) { + return serialise((std::stringstream() << "Key" << keyCount).str()); +} + +inline Value get_value(int64_t keyCount, int64_t valueCount) { + return serialise( + (std::stringstream() << "Key" << keyCount << "Data" << valueCount).str()); +} + +} // namespace bb::lmdblib \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_cursor.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_cursor.cpp similarity index 92% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_cursor.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_cursor.cpp index 809d9bd0dffb..b8c9958e9926 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_cursor.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_cursor.cpp @@ -1,7 +1,7 @@ -#include "barretenberg/lmdblib/lmdb_cursor.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/lmdblib/lmdb_read_transaction.hpp" -#include "barretenberg/lmdblib/queries.hpp" +#include "lmdblib/lmdb_cursor.hpp" +#include "lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_read_transaction.hpp" +#include "lmdblib/queries.hpp" #include "lmdb.h" #include diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_cursor.hpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_cursor.hpp similarity index 97% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_cursor.hpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_cursor.hpp index e1a0a1d22b6a..047d732c1f18 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_cursor.hpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_cursor.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/lmdblib/types.hpp" +#include "lmdblib/types.hpp" #include "lmdb.h" #include #include diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_database.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_database.cpp similarity index 79% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_database.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_database.cpp index 27c910aad7d8..4d089e77ca91 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_database.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_database.cpp @@ -1,9 +1,9 @@ -#include "barretenberg/lmdblib/lmdb_database.hpp" -#include "barretenberg/lmdblib/lmdb_db_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/lmdblib/lmdb_read_transaction.hpp" -#include "barretenberg/lmdblib/types.hpp" +#include "lmdblib/lmdb_database.hpp" +#include "lmdblib/lmdb_db_transaction.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_read_transaction.hpp" +#include "lmdblib/types.hpp" #include "lmdb.h" #include @@ -59,7 +59,8 @@ DBStats LMDBDatabase::get_stats(LMDBReadTransaction& tx) { MDB_stat stat; call_lmdb_func(mdb_stat, tx.underlying(), underlying(), &stat); - return DBStats(name(), stat); + uint64_t totalUsedSize = stat.ms_psize * (stat.ms_branch_pages + stat.ms_leaf_pages + stat.ms_overflow_pages); + return DBStats(name(), stat.ms_entries, totalUsedSize); } } // namespace bb::lmdblib diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_database.hpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_database.hpp similarity index 93% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_database.hpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_database.hpp index 2f8960647584..21ec77a98fc4 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_database.hpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_database.hpp @@ -1,6 +1,6 @@ #pragma once -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/types.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/types.hpp" namespace bb::lmdblib { diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_db_transaction.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_db_transaction.cpp similarity index 87% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_db_transaction.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_db_transaction.cpp index 549e3c78baad..ee9bd31e0164 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_db_transaction.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_db_transaction.cpp @@ -1,6 +1,6 @@ -#include "barretenberg/lmdblib/lmdb_db_transaction.hpp" +#include "lmdblib/lmdb_db_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_helpers.hpp" #include namespace bb::lmdblib { diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_db_transaction.hpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_db_transaction.hpp similarity index 94% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_db_transaction.hpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_db_transaction.hpp index e0fd2a3ad25c..38292930cbe4 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_db_transaction.hpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_db_transaction.hpp @@ -1,5 +1,5 @@ #pragma once -#include "barretenberg/lmdblib/lmdb_transaction.hpp" +#include "lmdblib/lmdb_transaction.hpp" namespace bb::lmdblib { diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_environment.cpp similarity index 95% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_environment.cpp index ab712c7fa2b5..e786a2628ce9 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_environment.cpp @@ -1,5 +1,5 @@ -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_helpers.hpp" #include "lmdb.h" #include #include diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.hpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_environment.hpp similarity index 100% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.hpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_environment.hpp diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.test.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_environment.test.cpp similarity index 95% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.test.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_environment.test.cpp index f196a20765d3..398fc1769815 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_environment.test.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_environment.test.cpp @@ -14,13 +14,13 @@ #include "barretenberg/common/serialize.hpp" #include "barretenberg/common/streams.hpp" #include "barretenberg/common/test.hpp" -#include "barretenberg/lmdblib/fixtures.hpp" -#include "barretenberg/lmdblib/lmdb_database.hpp" -#include "barretenberg/lmdblib/lmdb_db_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_read_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_write_transaction.hpp" -#include "barretenberg/lmdblib/queries.hpp" +#include "lmdblib/fixtures.hpp" +#include "lmdblib/lmdb_database.hpp" +#include "lmdblib/lmdb_db_transaction.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_read_transaction.hpp" +#include "lmdblib/lmdb_write_transaction.hpp" +#include "lmdblib/queries.hpp" using namespace bb::lmdblib; diff --git a/native-packages/lmdblib/cpp/src/lmdblib/lmdb_helpers.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_helpers.cpp new file mode 100644 index 000000000000..996aa77910d3 --- /dev/null +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_helpers.cpp @@ -0,0 +1,61 @@ +#include "lmdblib/lmdb_helpers.hpp" +#include "lmdb.h" +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#include +#define htole64(x) OSSwapHostToLittleInt64(x) +#define le64toh(x) OSSwapLittleToHostInt64(x) +#else +#include +#endif + +namespace bb::lmdblib { +void throw_error(const std::string &errorString, int error) { + std::stringstream ss; + ss << errorString << ": " << error << " - " << mdb_strerror(error) + << std::endl; + throw std::runtime_error(ss.str()); +} + +std::vector serialise_key(uint8_t key) { return {key}; } + +void deserialise_key(void *data, uint8_t &key) { + uint8_t *p = static_cast(data); + key = *p; +} + +// 64 bit integers are stored in little endian byte order +std::vector serialise_key(uint64_t key) { + uint64_t le = htole64(key); + const uint8_t *p = reinterpret_cast(&le); + return std::vector(p, p + sizeof(key)); +} + +void deserialise_key(void *data, uint64_t &key) { + uint64_t le = 0; + std::memcpy(&le, data, sizeof(le)); + key = le64toh(le); +} + +int size_cmp(const MDB_val *a, const MDB_val *b) { + if (a->mv_size < b->mv_size) { + return -1; + } + return (a->mv_size > b->mv_size) ? 1 : 0; +} + +std::vector mdb_val_to_vector(const MDB_val &dbVal) { + const uint8_t *p = static_cast(dbVal.mv_data); + return std::vector(p, p + dbVal.mv_size); +} + +void copy_to_vector(const MDB_val &dbVal, std::vector &target) { + std::vector temp = mdb_val_to_vector(dbVal); + target.swap(temp); +} +} // namespace bb::lmdblib \ No newline at end of file diff --git a/native-packages/lmdblib/cpp/src/lmdblib/lmdb_helpers.hpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_helpers.hpp new file mode 100644 index 000000000000..a7c9fa25ccd8 --- /dev/null +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_helpers.hpp @@ -0,0 +1,83 @@ + +#pragma once +#include "lmdb.h" +#include +#include +#include +#include +namespace bb::lmdblib { +// Concatenate args into a string (replaces barretenberg's common `format` +// helper). +template std::string format(Args &&...args) { + std::ostringstream os; + (os << ... << args); + return os.str(); +} + +void throw_error(const std::string &errorString, int error); + +int size_cmp(const MDB_val *a, const MDB_val *b); + +std::vector serialise_key(uint8_t key); +std::vector serialise_key(uint64_t key); + +void deserialise_key(void *data, uint8_t &key); +void deserialise_key(void *data, uint64_t &key); + +// Fixed-width "field-like" keys (e.g. a 32-byte field element): serialised as +// the type's raw little-endian bytes. Templated so lmdblib needs no concrete +// big-int type from barretenberg; callers (e.g. the merkle trees) instantiate +// it with their own key type. The byte layout is unchanged from the old uint256 +// overload. +template std::vector serialise_key(const T &key) { + std::vector buf(sizeof(T)); + std::memcpy(buf.data(), &key, sizeof(T)); + return buf; +} + +template void deserialise_key(void *data, T &key) { + std::memcpy(&key, data, sizeof(T)); +} + +template int value_cmp(const MDB_val *a, const MDB_val *b) { + T lhs; + T rhs; + deserialise_key(a->mv_data, lhs); + deserialise_key(b->mv_data, rhs); + if (lhs < rhs) { + return -1; + } + if (lhs > rhs) { + return 1; + } + return 0; +} + +std::vector mdb_val_to_vector(const MDB_val &dbVal); +void copy_to_vector(const MDB_val &dbVal, std::vector &target); + +template +bool call_lmdb_func(int (*f)(TArgs...), TArgs... args) { + int error = f(args...); + return error == 0; +} + +template +int call_lmdb_func_with_return(int (*f)(TArgs...), TArgs... args) { + return f(args...); +} + +template +void call_lmdb_func(const std::string &errorString, int (*f)(TArgs...), + TArgs... args) { + int error = f(args...); + if (error != 0) { + throw_error(errorString, error); + } +} + +template +void call_lmdb_func(void (*f)(TArgs...), TArgs... args) { + f(args...); +} +} // namespace bb::lmdblib \ No newline at end of file diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_read_transaction.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_read_transaction.cpp similarity index 64% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_read_transaction.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_read_transaction.cpp index 51a8c7f754cd..a20a98fa1357 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_read_transaction.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_read_transaction.cpp @@ -1,6 +1,6 @@ -#include "barretenberg/lmdblib/lmdb_read_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_read_transaction.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_helpers.hpp" #include namespace bb::lmdblib { diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_read_transaction.hpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_read_transaction.hpp similarity index 75% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_read_transaction.hpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_read_transaction.hpp index 298651d10792..d1f55aa82c46 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_read_transaction.hpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_read_transaction.hpp @@ -1,10 +1,8 @@ #pragma once -#include "barretenberg/common/serialize.hpp" -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/lmdblib/lmdb_transaction.hpp" -#include "barretenberg/lmdblib/queries.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_transaction.hpp" +#include "lmdblib/queries.hpp" #include #include #include diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_store.cpp similarity index 96% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_store.cpp index 15165a93d910..ddc9c9632513 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_store.cpp @@ -1,9 +1,9 @@ -#include "barretenberg/lmdblib/lmdb_store.hpp" -#include "barretenberg/lmdblib/lmdb_database.hpp" -#include "barretenberg/lmdblib/lmdb_db_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_store_base.hpp" -#include "barretenberg/lmdblib/lmdb_write_transaction.hpp" -#include "barretenberg/lmdblib/types.hpp" +#include "lmdblib/lmdb_store.hpp" +#include "lmdblib/lmdb_database.hpp" +#include "lmdblib/lmdb_db_transaction.hpp" +#include "lmdblib/lmdb_store_base.hpp" +#include "lmdblib/lmdb_write_transaction.hpp" +#include "lmdblib/types.hpp" #include "lmdb.h" #include #include diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store.hpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_store.hpp similarity index 86% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store.hpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_store.hpp index 299dc9f65c12..06d5431bfedd 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store.hpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_store.hpp @@ -1,13 +1,13 @@ #pragma once -#include "barretenberg/lmdblib/lmdb_cursor.hpp" -#include "barretenberg/lmdblib/lmdb_database.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_read_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_store_base.hpp" -#include "barretenberg/lmdblib/lmdb_write_transaction.hpp" -#include "barretenberg/lmdblib/queries.hpp" -#include "barretenberg/lmdblib/types.hpp" +#include "lmdblib/lmdb_cursor.hpp" +#include "lmdblib/lmdb_database.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_read_transaction.hpp" +#include "lmdblib/lmdb_store_base.hpp" +#include "lmdblib/lmdb_write_transaction.hpp" +#include "lmdblib/queries.hpp" +#include "lmdblib/types.hpp" #include #include #include diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store.test.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_store.test.cpp similarity index 99% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store.test.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_store.test.cpp index 75c677adc14b..79e8f1c3d357 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store.test.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_store.test.cpp @@ -15,16 +15,16 @@ #include "barretenberg/common/serialize.hpp" #include "barretenberg/common/streams.hpp" #include "barretenberg/common/test.hpp" -#include "barretenberg/lmdblib/fixtures.hpp" -#include "barretenberg/lmdblib/lmdb_cursor.hpp" -#include "barretenberg/lmdblib/lmdb_database.hpp" -#include "barretenberg/lmdblib/lmdb_db_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_read_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_store.hpp" -#include "barretenberg/lmdblib/lmdb_write_transaction.hpp" -#include "barretenberg/lmdblib/queries.hpp" -#include "barretenberg/lmdblib/types.hpp" +#include "lmdblib/fixtures.hpp" +#include "lmdblib/lmdb_cursor.hpp" +#include "lmdblib/lmdb_database.hpp" +#include "lmdblib/lmdb_db_transaction.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_read_transaction.hpp" +#include "lmdblib/lmdb_store.hpp" +#include "lmdblib/lmdb_write_transaction.hpp" +#include "lmdblib/queries.hpp" +#include "lmdblib/types.hpp" using namespace bb::lmdblib; diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store_base.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_store_base.cpp similarity index 97% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store_base.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_store_base.cpp index 459510da23bb..11fe1983ba9b 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store_base.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_store_base.cpp @@ -1,4 +1,4 @@ -#include "barretenberg/lmdblib/lmdb_store_base.hpp" +#include "lmdblib/lmdb_store_base.hpp" namespace bb::lmdblib { LMDBStoreBase::LMDBStoreBase( diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store_base.hpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_store_base.hpp similarity index 83% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store_base.hpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_store_base.hpp index fd03e29c4ea4..73e712f12231 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_store_base.hpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_store_base.hpp @@ -1,9 +1,9 @@ #pragma once -#include "barretenberg/lmdblib/lmdb_db_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_read_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_write_transaction.hpp" +#include "lmdblib/lmdb_db_transaction.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_read_transaction.hpp" +#include "lmdblib/lmdb_write_transaction.hpp" namespace bb::lmdblib { class LMDBStoreBase { diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_transaction.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_transaction.cpp similarity index 88% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_transaction.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_transaction.cpp index 45b1c9d1c2d2..1eee1c74063e 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_transaction.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_transaction.cpp @@ -1,6 +1,6 @@ -#include "barretenberg/lmdblib/lmdb_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_transaction.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_helpers.hpp" #include #include diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_transaction.hpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_transaction.hpp similarity index 97% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_transaction.hpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_transaction.hpp index d5bc5c4b0198..02389cddba9a 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_transaction.hpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_transaction.hpp @@ -1,6 +1,6 @@ #pragma once -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/queries.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/queries.hpp" #include "lmdb.h" #include #include diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_write_transaction.cpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_write_transaction.cpp similarity index 83% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_write_transaction.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_write_transaction.cpp index b4d3151ce03e..893cf92c3037 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_write_transaction.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_write_transaction.cpp @@ -1,11 +1,11 @@ -#include "barretenberg/lmdblib/lmdb_write_transaction.hpp" +#include "lmdblib/lmdb_write_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_database.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/lmdblib/queries.hpp" +#include "lmdblib/lmdb_database.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_helpers.hpp" +#include "lmdblib/queries.hpp" #include "lmdb.h" #include diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_write_transaction.hpp b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_write_transaction.hpp similarity index 89% rename from barretenberg/cpp/src/barretenberg/lmdblib/lmdb_write_transaction.hpp rename to native-packages/lmdblib/cpp/src/lmdblib/lmdb_write_transaction.hpp index 71674409b6fc..471aed286574 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/lmdb_write_transaction.hpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/lmdb_write_transaction.hpp @@ -1,11 +1,9 @@ #pragma once -#include "barretenberg/common/serialize.hpp" -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/lmdblib/lmdb_database.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/lmdblib/lmdb_transaction.hpp" -#include "barretenberg/lmdblib/queries.hpp" +#include "lmdblib/lmdb_database.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_transaction.hpp" +#include "lmdblib/queries.hpp" #include "lmdb.h" #include #include diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/queries.cpp b/native-packages/lmdblib/cpp/src/lmdblib/queries.cpp similarity index 97% rename from barretenberg/cpp/src/barretenberg/lmdblib/queries.cpp rename to native-packages/lmdblib/cpp/src/lmdblib/queries.cpp index 9bbb54ee8cb4..ee770945e392 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/queries.cpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/queries.cpp @@ -1,8 +1,8 @@ -#include "barretenberg/lmdblib/queries.hpp" -#include "barretenberg/lmdblib/lmdb_cursor.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/lmdblib/lmdb_write_transaction.hpp" -#include "barretenberg/lmdblib/types.hpp" +#include "lmdblib/queries.hpp" +#include "lmdblib/lmdb_cursor.hpp" +#include "lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_write_transaction.hpp" +#include "lmdblib/types.hpp" #include "lmdb.h" #include #include diff --git a/barretenberg/cpp/src/barretenberg/lmdblib/queries.hpp b/native-packages/lmdblib/cpp/src/lmdblib/queries.hpp similarity index 99% rename from barretenberg/cpp/src/barretenberg/lmdblib/queries.hpp rename to native-packages/lmdblib/cpp/src/lmdblib/queries.hpp index 9f5ac1d09f2b..29af44889b1c 100644 --- a/barretenberg/cpp/src/barretenberg/lmdblib/queries.hpp +++ b/native-packages/lmdblib/cpp/src/lmdblib/queries.hpp @@ -1,8 +1,8 @@ #pragma once -#include "barretenberg/lmdblib/lmdb_database.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/lmdblib/types.hpp" +#include "lmdblib/lmdb_database.hpp" +#include "lmdblib/lmdb_helpers.hpp" +#include "lmdblib/types.hpp" #include "lmdb.h" #include #include diff --git a/native-packages/lmdblib/cpp/src/lmdblib/serialization.hpp b/native-packages/lmdblib/cpp/src/lmdblib/serialization.hpp new file mode 100644 index 000000000000..af6494515fa0 --- /dev/null +++ b/native-packages/lmdblib/cpp/src/lmdblib/serialization.hpp @@ -0,0 +1,14 @@ +#pragma once + +// msgpack-c entrypoint for lmdblib. We use msgpack-c directly (no barretenberg +// serialization framework). The AztecProtocol msgpack-c fork expects the +// includer to provide THROW/RETHROW macros; define them to standard exception +// handling. +#ifndef THROW +#define THROW throw +#endif +#ifndef RETHROW +#define RETHROW throw +#endif + +#include diff --git a/native-packages/lmdblib/cpp/src/lmdblib/types.hpp b/native-packages/lmdblib/cpp/src/lmdblib/types.hpp new file mode 100644 index 000000000000..fd092160b18b --- /dev/null +++ b/native-packages/lmdblib/cpp/src/lmdblib/types.hpp @@ -0,0 +1,64 @@ +#pragma once + +#include "lmdblib/serialization.hpp" +#include +#include +#include +#include +#include +namespace bb::lmdblib { +using Key = std::vector; +using Value = std::vector; +using KeysVector = std::vector; +using ValuesVector = std::vector; +using KeyValuesPair = std::pair; +using OptionalValues = std::optional; +using OptionalValuesVector = std::vector; +using KeyDupValuesVector = std::vector; +using KeyOptionalValuesPair = std::pair; +using KeyOptionalValuesVector = std::vector; + +/** + * @brief Storage statistics for a single named lmdb database. + * + * Plain wire value type owned by lmdblib (it is what lmdb's get_stats reports). + */ +struct DBStats { + std::string name; + uint64_t numDataItems; + uint64_t totalUsedSize; + + DBStats() = default; + DBStats(const DBStats &other) = default; + DBStats(DBStats &&other) noexcept { *this = std::move(other); } + ~DBStats() = default; + DBStats(const std::string &name, uint64_t numDataItems, + uint64_t totalUsedSize) + : name(name), numDataItems(numDataItems), totalUsedSize(totalUsedSize) {} + + MSGPACK_DEFINE_MAP(name, numDataItems, totalUsedSize) + + bool operator==(const DBStats &other) const { + return name == other.name && numDataItems == other.numDataItems && + totalUsedSize == other.totalUsedSize; + } + + DBStats &operator=(const DBStats &other) = default; + + DBStats &operator=(DBStats &&other) noexcept { + if (this != &other) { + name = std::move(other.name); + numDataItems = other.numDataItems; + totalUsedSize = other.totalUsedSize; + } + return *this; + } + + friend std::ostream &operator<<(std::ostream &os, const DBStats &stats) { + os << "DB " << stats.name << ", num items: " << stats.numDataItems + << ", total used size: " << stats.totalUsedSize; + return os; + } +}; + +} // namespace bb::lmdblib \ No newline at end of file diff --git a/native-packages/wsdb/.gitignore b/native-packages/wsdb/.gitignore new file mode 100644 index 000000000000..f80c50825486 --- /dev/null +++ b/native-packages/wsdb/.gitignore @@ -0,0 +1,12 @@ +ts/ +cpp/build/ +# ipc-codegen emits the server dispatch into a generated/ subdir at build time. +**/generated/ +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/sdks +!.yarn/versions +node_modules +*.log diff --git a/native-packages/wsdb/.rebuild_patterns b/native-packages/wsdb/.rebuild_patterns new file mode 100644 index 000000000000..6c2d11b6e8ed --- /dev/null +++ b/native-packages/wsdb/.rebuild_patterns @@ -0,0 +1,8 @@ +^native-packages/wsdb/bootstrap\.sh$ +^native-packages/wsdb/package\.json$ +^native-packages/wsdb/\.rebuild_patterns$ +^native-packages/wsdb/wsdb_schema\.jsonc$ +^native-packages/wsdb/cpp/ +^native-packages/lmdblib/ +^ipc-codegen/src/.*\.ts$ +^ipc-codegen/templates/ diff --git a/native-packages/wsdb/.yarnrc.yml b/native-packages/wsdb/.yarnrc.yml new file mode 100644 index 000000000000..3186f3f0795a --- /dev/null +++ b/native-packages/wsdb/.yarnrc.yml @@ -0,0 +1 @@ +nodeLinker: node-modules diff --git a/wsdb/bootstrap.sh b/native-packages/wsdb/bootstrap.sh similarity index 53% rename from wsdb/bootstrap.sh rename to native-packages/wsdb/bootstrap.sh index 577cc1e5779e..55c83ba6e9af 100755 --- a/wsdb/bootstrap.sh +++ b/native-packages/wsdb/bootstrap.sh @@ -2,65 +2,59 @@ source $(git rev-parse --show-toplevel)/ci3/source_bootstrap ROOT=$(git rev-parse --show-toplevel) +PKG="$ROOT/native-packages/wsdb" WSDB_BINARY=aztec-wsdb +# This package owns the wire contract; both the TS client and the C++ server +# codegen from this schema. +WSDB_SCHEMA="$PKG/wsdb_schema.jsonc" hash=$(hash_str \ - $(../barretenberg/cpp/bootstrap.sh hash) \ - $(../ipc-runtime/bootstrap.sh hash) \ + $(../../barretenberg/cpp/bootstrap.sh hash) \ + $(../../ipc-runtime/bootstrap.sh hash) \ + $(../lmdblib/bootstrap.sh hash) \ $(cache_content_hash .rebuild_patterns)) function generate_ts_package { node --experimental-strip-types --experimental-transform-types --no-warnings \ "$ROOT/ipc-codegen/src/generate.ts" \ - --schema "$ROOT/barretenberg/cpp/src/barretenberg/wsdb/wsdb_schema.jsonc" \ + --schema "$WSDB_SCHEMA" \ --lang ts \ --client \ - --out "$ROOT/wsdb/ts/src/generated" \ - --package "$ROOT/wsdb/ts" \ + --out "$PKG/ts/src/generated" \ + --package "$PKG/ts" \ --package-name @aztec/wsdb \ --binary-name "$WSDB_BINARY" \ --package-transports uds,shm \ --package-ipc-path-args 'msgpack,run,--input,{path}' } -function copy_native { +# Build the standalone aztec-wsdb binary against the prebuilt barretenberg +# artifacts (default preset -> barretenberg/cpp/build). bb is never rebuilt here. +function build_native { + local build_dir="cpp/build" + CC=$(which clang) CXX=$(which clang++) cmake -S cpp -B "$build_dir" -G Ninja >/dev/null + cmake --build "$build_dir" --target "$WSDB_BINARY" local target_dir="ts/build/$(arch)-$(os)" mkdir -p "$target_dir" - cp "$ROOT/barretenberg/cpp/build/bin/$WSDB_BINARY" "$target_dir/$WSDB_BINARY" -} - -function copy_cross { - if [ -n "${1:-}" ]; then - local cross_arch="$1" - mkdir -p "ts/build/$cross_arch" - cp "$ROOT/barretenberg/cpp/build-$cross_arch/bin/$WSDB_BINARY" "ts/build/$cross_arch/$WSDB_BINARY" - elif semver check "${REF_NAME:-}" && [ "$(arch)" == "amd64" ]; then - for cross_arch in arm64-linux amd64-macos arm64-macos; do - mkdir -p "ts/build/$cross_arch" - cp "$ROOT/barretenberg/cpp/build-$cross_arch/bin/$WSDB_BINARY" "ts/build/$cross_arch/$WSDB_BINARY" - done - else - echo "This task is expected to be run with an explicit arch or in an x86 release context." - fi + cp "$build_dir/bin/$WSDB_BINARY" "$target_dir/$WSDB_BINARY" } function build { echo_header "wsdb build" generate_ts_package - copy_native + build_native npm_install_deps yarn build (cd ts && ./scripts/prepare_arch_packages.sh "$(arch)-$(os)=build/$(arch)-$(os)/$WSDB_BINARY") } function clean { - rm -rf ts node_modules + rm -rf ts node_modules cpp/build } function release { generate_ts_package - copy_native - copy_cross + build_native npm_install_deps yarn build (cd ts && ./scripts/prepare_arch_packages.sh) @@ -70,7 +64,7 @@ function release { (cd ts && retry "deploy_npm ${REF_NAME#v}") } -export -f generate_ts_package copy_native copy_cross build clean release +export -f generate_ts_package build_native build clean release case "$cmd" in "") diff --git a/native-packages/wsdb/cpp/CMakeLists.txt b/native-packages/wsdb/cpp/CMakeLists.txt new file mode 100644 index 000000000000..50fcefe77fdc --- /dev/null +++ b/native-packages/wsdb/cpp/CMakeLists.txt @@ -0,0 +1,257 @@ +# Standalone build for the aztec-wsdb world-state database service. +# +# This package owns the wsdb service: the world_state engine, the persistent +# content-addressed merkle storage, the lmdb tree store, and the IPC server. +# It depends on barretenberg only as a *prebuilt* artifact: bb is assumed to be +# already built in place (default preset -> barretenberg/cpp/build), and this +# project links the resulting static archives. It never (re)builds bb. +# +# Link target: the full libbarretenberg.a (for the generic merkle core), plus the +# lmdblib sibling package's liblmdblib.a (the lmdb C++ wrapper, which barretenberg +# no longer hosts). bb-external is intentionally NOT usable here -- it excludes +# crypto_merkle_tree_objects (see barretenberg/cpp/src/CMakeLists.txt). + +cmake_minimum_required(VERSION 3.24) +# C is enabled so CMAKE_C_COMPILER is populated for lmdb's ExternalProject build. +project(aztec_wsdb CXX C) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + +# Match barretenberg's prebuilt archive build (Release -> -O3 -DNDEBUG). bb headers +# (inline/template crypto + field code) are compiled into BOTH bb's Release archive +# and this build; compiling them here without NDEBUG/-O3 is an ODR violation that can +# silently diverge inlined results (e.g. tree leaf hashes). Default to Release. +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif() + +# --------------------------------------------------------------------------- +# Locate the prebuilt barretenberg tree. +# --------------------------------------------------------------------------- +get_filename_component(REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../.." ABSOLUTE) +set(BB_SRC "${REPO_ROOT}/barretenberg/cpp/src") +set(BB_BUILD "${REPO_ROOT}/barretenberg/cpp/build" CACHE PATH "barretenberg build dir") +set(IPC_RUNTIME_SRC "${REPO_ROOT}/ipc-runtime/cpp") +set(IPC_CODEGEN_DIR "${REPO_ROOT}/ipc-codegen") + +set(BB_LIB "${BB_BUILD}/lib/libbarretenberg.a") +# libbarretenberg.a deliberately excludes the env module (logstr / throw_or_abort_impl / +# hardware_concurrency) so consumers supply their own runtime. As a native command we +# link bb's env implementation, matching what the old in-bb aztec-wsdb target did. +set(ENV_LIB "${BB_BUILD}/lib/libenv.a") +set(IPC_RUNTIME_LIB "${BB_BUILD}/lib/libipc_runtime.a") + +# lmdblib sibling package: headers + the prebuilt wrapper archive (the lmdb C++ +# wrapper that barretenberg no longer hosts). +set(LMDBLIB_PKG "${REPO_ROOT}/native-packages/lmdblib") +set(LMDBLIB_SRC "${LMDBLIB_PKG}/cpp/src") +set(LMDBLIB_LIB "${LMDBLIB_PKG}/cpp/build/lib/liblmdblib.a") + +# We depend on barretenberg only for its *published* static archives (under +# ${BB_BUILD}/lib), which are the artifacts CI caches. bb's FetchContent build +# internals (${BB_BUILD}/_deps/...) are NOT cached, so we fetch and build our own +# copies of the third-party deps here, pinned to the exact commits bb uses so the +# headers/ABI line up with the prebuilt libbarretenberg.a. +foreach(artifact "${BB_LIB}" "${ENV_LIB}" "${IPC_RUNTIME_LIB}" "${LMDBLIB_LIB}") + if(NOT EXISTS "${artifact}") + message(FATAL_ERROR "Missing prebuilt artifact: ${artifact}\n" + "Build barretenberg (cd barretenberg/cpp && cmake --preset default && cd build && ninja) " + "and lmdblib (cd native-packages/lmdblib && ./bootstrap.sh) first.") + endif() +endforeach() + +# --------------------------------------------------------------------------- +# Third-party deps (own copies; commits must match barretenberg/cpp/cmake/*). +# --------------------------------------------------------------------------- +set(DEPS_PREFIX "${CMAKE_BINARY_DIR}/_deps") + +# lmdb: build our own liblmdb.a (pinned to bb's commit in cmake/lmdb.cmake). +include(ExternalProject) +set(LMDB_PREFIX "${DEPS_PREFIX}/lmdb") +set(LMDB_INCLUDE "${LMDB_PREFIX}/src/lmdb_repo/libraries/liblmdb") +set(LMDB_LIB "${LMDB_INCLUDE}/liblmdb.a") +ExternalProject_Add( + lmdb_repo + PREFIX ${LMDB_PREFIX} + DOWNLOAD_COMMAND + sh -c "mkdir -p ${LMDB_PREFIX}/src/lmdb_repo && cd ${LMDB_PREFIX}/src/lmdb_repo && git init --quiet && (git remote add origin https://github.com/LMDB/lmdb.git 2>/dev/null || true) && git fetch --depth 1 origin --quiet ddd0a773e2f44d38e4e31ec9ed81af81f4e4ccbb && git reset --quiet --hard FETCH_HEAD" + SOURCE_DIR ${LMDB_PREFIX}/src/lmdb_repo + BUILD_IN_SOURCE YES + CONFIGURE_COMMAND "" + BUILD_COMMAND ${CMAKE_COMMAND} -E env --unset=CFLAGS --unset=CXXFLAGS CC=${CMAKE_C_COMPILER}${CMAKE_C_COMPILER_ARG1} AR=${CMAKE_AR} make -e -C libraries/liblmdb XCFLAGS=-fPIC liblmdb.a + INSTALL_COMMAND "" + UPDATE_COMMAND "" + BUILD_BYPRODUCTS ${LMDB_LIB} +) +add_library(lmdb STATIC IMPORTED GLOBAL) +add_dependencies(lmdb lmdb_repo) +set_target_properties(lmdb PROPERTIES IMPORTED_LOCATION ${LMDB_LIB}) + +# msgpack-c: header-only; fetch for includes (pinned to bb's commit in cmake/msgpack.cmake). +set(MSGPACK_PREFIX "${DEPS_PREFIX}/msgpack-c") +set(MSGPACK_INCLUDE "${MSGPACK_PREFIX}/src/msgpack-c/include") +ExternalProject_Add( + msgpack-c + PREFIX ${MSGPACK_PREFIX} + DOWNLOAD_COMMAND + sh -c "mkdir -p ${MSGPACK_PREFIX}/src/msgpack-c && cd ${MSGPACK_PREFIX}/src/msgpack-c && git init --quiet && (git remote add origin https://github.com/AztecProtocol/msgpack-c.git 2>/dev/null || true) && git fetch --depth 1 origin --quiet c0334576ed657fb3b3c49e8e61402989fb84146d && git reset --quiet --hard FETCH_HEAD" + SOURCE_DIR ${MSGPACK_PREFIX}/src/msgpack-c + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + UPDATE_COMMAND "" +) + +# tracy: header-only here (we only include its headers); pinned to bb's commit in cmake/tracy.cmake. +set(TRACY_SOURCE_DIR "${DEPS_PREFIX}/tracy-src") +execute_process( + COMMAND sh -c "mkdir -p ${TRACY_SOURCE_DIR} && cd ${TRACY_SOURCE_DIR} && git init --quiet && (git remote add origin https://github.com/wolfpld/tracy.git 2>/dev/null || true) && git fetch --depth 1 origin --quiet 5d542dc09f3d9378d005092a4ad446bd405f819a && git reset --quiet --hard FETCH_HEAD" + RESULT_VARIABLE tracy_result +) +if(tracy_result) + message(FATAL_ERROR "Failed to download Tracy") +endif() +set(TRACY_INCLUDE "${TRACY_SOURCE_DIR}/public") + +# Match the ABI-affecting compile flags of the bb build. bb is compiled with +# clang and -march=skylake; mixing toolchains/arch against bb's prebuilt static +# archives risks subtle ODR/template-instantiation mismatches, so match it. +add_compile_definitions(MSGPACK_NO_BOOST MSGPACK_USE_STD_VARIANT_ADAPTOR NO_PAR_ALGOS) +add_compile_options(-march=skylake -fconstexpr-steps=100000000 -fbracket-depth=1024) +add_compile_options(-Wno-deprecated-declarations -Wno-missing-field-initializers) +# bb's archives are built with libc++ (std::__1:: symbols). The system clang here +# links libstdc++ by default, leaving those symbols undefined, so force libc++ for +# both compile and link to match bb's ABI. +add_compile_options(-stdlib=libc++) +# Statically link libc++/libc++abi/libgcc so aztec-wsdb is self-contained (no +# libc++.so.1 runtime dependency), matching bb's own aztec-wsdb. Otherwise the +# binary fails to launch (exit 127) in minimal environments — e.g. a scaffolded +# user project — that don't ship the libc++ runtime. +add_link_options(-stdlib=libc++ -static-libstdc++ -static-libgcc) + +include_directories( + ${CMAKE_CURRENT_SOURCE_DIR}/src + ${BB_SRC} + ${LMDBLIB_SRC} + ${MSGPACK_INCLUDE} + ${TRACY_INCLUDE} + ${LMDB_INCLUDE} + ${IPC_RUNTIME_SRC} +) + +# --------------------------------------------------------------------------- +# Codegen the wsdb server dispatch from the local schema (this package owns the +# wire contract). bb's client codegen reads the same schema from this path. +# --------------------------------------------------------------------------- +set(WSDB_SCHEMA ${CMAKE_CURRENT_SOURCE_DIR}/../wsdb_schema.jsonc) +set(WSDB_GEN_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/wsdb/generated) +set(WSDB_GEN_OUTPUTS + ${WSDB_GEN_DIR}/wsdb_dispatch.hpp + ${WSDB_GEN_DIR}/wsdb_ipc_server.hpp + ${WSDB_GEN_DIR}/wsdb_types.hpp + ${WSDB_GEN_DIR}/ipc_codegen/msgpack_adaptor.hpp + ${WSDB_GEN_DIR}/ipc_codegen/named_union.hpp + ${WSDB_GEN_DIR}/ipc_codegen/throw.hpp +) +file(GLOB_RECURSE IPC_CODEGEN_SRC + ${IPC_CODEGEN_DIR}/src/*.ts + ${IPC_CODEGEN_DIR}/templates/cpp/*.hpp +) +add_custom_command( + OUTPUT ${WSDB_GEN_OUTPUTS} + COMMAND node --experimental-strip-types --experimental-transform-types --no-warnings + ${IPC_CODEGEN_DIR}/src/generate.ts + --schema ${WSDB_SCHEMA} + --lang cpp + --out ${WSDB_GEN_DIR} + --server + --cpp-namespace bb::wsdb + --cpp-include-dir wsdb/generated + DEPENDS ${WSDB_SCHEMA} ${IPC_CODEGEN_SRC} + COMMENT "Generating WSDB IPC server dispatch from wsdb_schema.jsonc" + VERBATIM +) +add_custom_target(wsdb_ipc_server_generated DEPENDS ${WSDB_GEN_OUTPUTS}) + +# --------------------------------------------------------------------------- +# wsdb core: world_state + persistent merkle storage + server handlers. +# --------------------------------------------------------------------------- +set(WSDB_CORE_SOURCES + src/world_state/types.cpp + src/world_state/world_state.cpp + src/merkle_tree/lmdb_store/lmdb_tree_store.cpp + src/wsdb/wsdb_handlers.cpp + src/wsdb/wsdb_ipc_server.cpp + src/wsdb/cli.cpp +) +add_library(wsdb_core STATIC ${WSDB_CORE_SOURCES}) +add_dependencies(wsdb_core wsdb_ipc_server_generated lmdb_repo msgpack-c) +# Wrap libbarretenberg.a + libenv.a in --start-group/--end-group: bb's archive +# references env symbols (logstr, throw_or_abort_impl) and the link order between +# the two is not a clean DAG, so let the linker resolve them mutually. +target_link_libraries(wsdb_core PUBLIC + -Wl,--start-group ${BB_LIB} ${ENV_LIB} ${LMDBLIB_LIB} -Wl,--end-group + ${IPC_RUNTIME_LIB} ${LMDB_LIB} pthread) + +# --------------------------------------------------------------------------- +# aztec-wsdb binary. +# --------------------------------------------------------------------------- +add_executable(aztec-wsdb src/wsdb/main.cpp) +target_link_libraries(aztec-wsdb PRIVATE wsdb_core) + +# --------------------------------------------------------------------------- +# Tests. The equality test links MemoryMerkleDB (the reference impl, which stays +# in bb's vm2_sim module -> libvm2_sim.a) against a real WorldState (local). It +# can no longer live in vm2_tests since that target no longer links world_state. +# --------------------------------------------------------------------------- +option(WSDB_BUILD_TESTS "Build wsdb C++ tests" ON) +if(WSDB_BUILD_TESTS) + set(VM2_SIM_LIB "${BB_BUILD}/lib/libvm2_sim.a") + set(GTEST_LIB "${BB_BUILD}/lib/libgtest.a") + set(GMOCK_LIB "${BB_BUILD}/lib/libgmock.a") + set(GMOCK_MAIN_LIB "${BB_BUILD}/lib/libgmock_main.a") + # gtest/gmock libs come from bb's published archives (cached), but their headers + # live under bb's uncached _deps. Fetch our own copy, pinned to bb's tag (cmake/gtest.cmake). + set(GTEST_PREFIX "${DEPS_PREFIX}/gtest-src") + ExternalProject_Add( + gtest_headers + PREFIX ${GTEST_PREFIX} + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG v1.13.0 + GIT_SHALLOW TRUE + SOURCE_DIR ${GTEST_PREFIX}/src/gtest + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + UPDATE_COMMAND "" + ) + set(GTEST_INCLUDE "${GTEST_PREFIX}/src/gtest/googletest/include") + set(GMOCK_INCLUDE "${GTEST_PREFIX}/src/gtest/googlemock/include") + + set(WSDB_TEST_SOURCES + src/merkle_tree/fixtures.test.cpp + src/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp + src/merkle_tree/node_store/content_addressed_cache.test.cpp + src/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp + src/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp + src/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp + src/world_state/world_state.test.cpp + src/wsdb/memory_merkle_db.test.cpp + ) + add_executable(wsdb_tests ${WSDB_TEST_SOURCES}) + add_dependencies(wsdb_tests wsdb_ipc_server_generated gtest_headers) + target_include_directories(wsdb_tests PRIVATE ${GTEST_INCLUDE} ${GMOCK_INCLUDE}) + # wsdb_core (WorldState + persistent merkle), vm2_sim (MemoryMerkleDB) and bb's + # archive reference each other's symbols, so group them all for mutual resolution. + target_link_libraries(wsdb_tests PRIVATE + -Wl,--start-group + $ ${VM2_SIM_LIB} ${BB_LIB} ${ENV_LIB} ${LMDBLIB_LIB} + -Wl,--end-group + ${IPC_RUNTIME_LIB} ${LMDB_LIB} pthread + ${GMOCK_MAIN_LIB} ${GMOCK_LIB} ${GTEST_LIB} + ) + add_dependencies(wsdb_tests wsdb_core) +endif() diff --git a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp b/native-packages/wsdb/cpp/src/benchmark/append_only_tree_bench/append_only_tree.bench.cpp similarity index 85% rename from barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp rename to native-packages/wsdb/cpp/src/benchmark/append_only_tree_bench/append_only_tree.bench.cpp index ed54fd5debc5..7604577f8df1 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp +++ b/native-packages/wsdb/cpp/src/benchmark/append_only_tree_bench/append_only_tree.bench.cpp @@ -1,11 +1,11 @@ #include "barretenberg/common/thread_pool.hpp" -#include "barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" -#include "barretenberg/crypto/merkle_tree/fixtures.hpp" +#include "merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" +#include "merkle_tree/fixtures.hpp" #include "barretenberg/crypto/merkle_tree/hash.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/array_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/node_store/array_store.hpp" +#include "merkle_tree/node_store/cached_content_addressed_tree_store.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/crypto/merkle_tree/signal.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" diff --git a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp b/native-packages/wsdb/cpp/src/benchmark/indexed_tree_bench/indexed_tree.bench.cpp similarity index 97% rename from barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp rename to native-packages/wsdb/cpp/src/benchmark/indexed_tree_bench/indexed_tree.bench.cpp index ec649651882e..390b47eea20b 100644 --- a/barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp +++ b/native-packages/wsdb/cpp/src/benchmark/indexed_tree_bench/indexed_tree.bench.cpp @@ -1,10 +1,10 @@ #include "barretenberg/common/thread_pool.hpp" -#include "barretenberg/crypto/merkle_tree/fixtures.hpp" +#include "merkle_tree/fixtures.hpp" #include "barretenberg/crypto/merkle_tree/hash.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp" +#include "merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/node_store/cached_content_addressed_tree_store.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/numeric/random/engine.hpp" #include diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp b/native-packages/wsdb/cpp/src/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp similarity index 99% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp index e119ae1606ad..0eff491b6a61 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp @@ -22,9 +22,10 @@ #include "barretenberg/common/thread_pool.hpp" #include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" +#include "merkle_tree/tree_db_stats.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "barretenberg/crypto/merkle_tree/tree_meta.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/crypto/merkle_tree/signal.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp b/native-packages/wsdb/cpp/src/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp similarity index 99% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp rename to native-packages/wsdb/cpp/src/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp index ef5f713561b6..b05d823766b8 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp @@ -1,20 +1,21 @@ -#include "barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" -#include "../fixtures.hpp" -#include "../memory_tree.hpp" -#include "../test_fixtures.hpp" +#include "merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" +#include "merkle_tree/tree_db_stats.hpp" +#include "merkle_tree/fixtures.hpp" +#include "barretenberg/crypto/merkle_tree/memory_tree.hpp" +#include "merkle_tree/test_fixtures.hpp" #include "barretenberg/common/streams.hpp" #include "barretenberg/common/test.hpp" #include "barretenberg/common/thread_pool.hpp" #include "barretenberg/crypto/merkle_tree/hash.hpp" #include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/array_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/node_store/array_store.hpp" +#include "merkle_tree/node_store/cached_content_addressed_tree_store.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/crypto/merkle_tree/signal.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_environment.hpp" #include "barretenberg/relations/relation_parameters.hpp" #include #include diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.hpp b/native-packages/wsdb/cpp/src/merkle_tree/fixtures.hpp similarity index 95% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/fixtures.hpp index b29045166ef2..10856491bfcf 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.hpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/fixtures.hpp @@ -7,7 +7,8 @@ #include #include "barretenberg/common/thread_pool.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/tree_db_stats.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" #include "barretenberg/numeric/random/engine.hpp" diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.test.cpp b/native-packages/wsdb/cpp/src/merkle_tree/fixtures.test.cpp similarity index 100% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.test.cpp rename to native-packages/wsdb/cpp/src/merkle_tree/fixtures.test.cpp diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp b/native-packages/wsdb/cpp/src/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp similarity index 99% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp index 1d547f089d0b..585e25927311 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp @@ -27,13 +27,13 @@ #include "barretenberg/common/assert.hpp" #include "barretenberg/common/thread_pool.hpp" #include "barretenberg/common/utils.hpp" -#include "barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" +#include "merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" #include "barretenberg/crypto/merkle_tree/hash.hpp" #include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/node_store/cached_content_addressed_tree_store.hpp" +#include "barretenberg/crypto/merkle_tree/tree_meta.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/crypto/merkle_tree/signal.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp b/native-packages/wsdb/cpp/src/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp similarity index 99% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp rename to native-packages/wsdb/cpp/src/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp index ce459dff63b9..83c7f292227e 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/indexed_tree/content_addressed_indexed_tree.test.cpp @@ -1,16 +1,17 @@ #include "./content_addressed_indexed_tree.hpp" -#include "../fixtures.hpp" -#include "../hash.hpp" -#include "../node_store/array_store.hpp" -#include "../nullifier_tree/nullifier_memory_tree.hpp" -#include "../test_fixtures.hpp" +#include "merkle_tree/fixtures.hpp" +#include "merkle_tree/tree_db_stats.hpp" +#include "barretenberg/crypto/merkle_tree/hash.hpp" +#include "merkle_tree/node_store/array_store.hpp" +#include "merkle_tree/nullifier_tree/nullifier_memory_tree.hpp" +#include "merkle_tree/test_fixtures.hpp" #include "barretenberg/common/streams.hpp" #include "barretenberg/common/test.hpp" #include "barretenberg/common/thread_pool.hpp" #include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/node_store/cached_content_addressed_tree_store.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/numeric/random/engine.hpp" diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp b/native-packages/wsdb/cpp/src/merkle_tree/lmdb_store/lmdb_tree_store.cpp similarity index 95% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp rename to native-packages/wsdb/cpp/src/merkle_tree/lmdb_store/lmdb_tree_store.cpp index 34bb07c30b75..744c4000ea45 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/lmdb_store/lmdb_tree_store.cpp @@ -4,18 +4,19 @@ // external_2: { status: not started, auditors: [], commit: } // ===================== -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" #include "barretenberg/common/serialize.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/lmdblib/lmdb_db_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/lmdblib/lmdb_store_base.hpp" #include "barretenberg/numeric/uint128/uint128.hpp" #include "barretenberg/numeric/uint256/uint256.hpp" #include "barretenberg/serialize/msgpack_impl.hpp" #include "lmdb_tree_store.hpp" +#include "lmdblib/lmdb_db_transaction.hpp" +#include "lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_store_base.hpp" +#include "merkle_tree/tree_db_stats.hpp" #include #include #include @@ -282,7 +283,11 @@ void LMDBTreeStore::decrement_node_reference_count(const fr& nodeHash, NodePaylo } if (--nodeData.ref == 0) { // std::cout << "Deleting node at " << nodeHash << std::endl; - tx.delete_value(nodeHash, *_nodeDatabase); + // Wrap in FrKeyType so the node key is serialised identically to write_node/read_node + // (canonical uint256 bytes). Passing the raw fr would bind serialise_key's generic + // template and emit Montgomery-form bytes, deleting the wrong node. + FrKeyType key(nodeHash); + tx.delete_value(key, *_nodeDatabase); return; } // std::cout << "Updating node at " << nodeHash << " ref is now " << nodeData.ref << std::endl; diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp b/native-packages/wsdb/cpp/src/merkle_tree/lmdb_store/lmdb_tree_store.hpp similarity index 95% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/lmdb_store/lmdb_tree_store.hpp index c916924f4e69..53b1974b4c25 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/lmdb_store/lmdb_tree_store.hpp @@ -7,17 +7,18 @@ #pragma once #include "barretenberg/common/log.hpp" #include "barretenberg/common/serialize.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/tree_db_stats.hpp" +#include "barretenberg/crypto/merkle_tree/tree_meta.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/lmdblib/lmdb_database.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" -#include "barretenberg/lmdblib/lmdb_read_transaction.hpp" -#include "barretenberg/lmdblib/lmdb_store_base.hpp" -#include "barretenberg/lmdblib/lmdb_write_transaction.hpp" +#include "lmdblib/lmdb_database.hpp" +#include "lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_read_transaction.hpp" +#include "lmdblib/lmdb_store_base.hpp" +#include "lmdblib/lmdb_write_transaction.hpp" #include "barretenberg/serialize/msgpack_impl.hpp" -#include "barretenberg/world_state/types.hpp" +#include "world_state/types.hpp" #include "lmdb.h" #include #include diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp b/native-packages/wsdb/cpp/src/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp similarity index 98% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp rename to native-packages/wsdb/cpp/src/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp index cbad34090212..77ca56025254 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp @@ -11,11 +11,11 @@ #include "barretenberg/common/serialize.hpp" #include "barretenberg/common/streams.hpp" #include "barretenberg/common/test.hpp" -#include "barretenberg/crypto/merkle_tree/fixtures.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" +#include "merkle_tree/fixtures.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/tree_meta.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_helpers.hpp" #include "barretenberg/numeric/random/engine.hpp" #include "barretenberg/numeric/uint128/uint128.hpp" #include "barretenberg/numeric/uint256/uint256.hpp" diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/array_store.hpp b/native-packages/wsdb/cpp/src/merkle_tree/node_store/array_store.hpp similarity index 98% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/array_store.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/node_store/array_store.hpp index a04918875c16..bd5463956148 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/array_store.hpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/node_store/array_store.hpp @@ -5,7 +5,7 @@ // ===================== #pragma once -#include "./tree_meta.hpp" +#include "barretenberg/crypto/merkle_tree/tree_meta.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/stdlib/primitives/field/field.hpp" #include diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp b/native-packages/wsdb/cpp/src/merkle_tree/node_store/cached_content_addressed_tree_store.hpp similarity index 99% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/node_store/cached_content_addressed_tree_store.hpp index e8c90065b16d..9feb2dfb76d8 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/node_store/cached_content_addressed_tree_store.hpp @@ -5,15 +5,16 @@ // ===================== #pragma once -#include "./tree_meta.hpp" +#include "barretenberg/crypto/merkle_tree/tree_meta.hpp" +#include "merkle_tree/tree_db_stats.hpp" #include "barretenberg/common/log.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/content_addressed_cache.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/node_store/content_addressed_cache.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/lmdblib/lmdb_transaction.hpp" +#include "lmdblib/lmdb_helpers.hpp" +#include "lmdblib/lmdb_transaction.hpp" #include "barretenberg/numeric/uint256/uint256.hpp" #include "barretenberg/serialize/msgpack.hpp" #include "barretenberg/stdlib/primitives/field/field.hpp" diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/content_addressed_cache.hpp b/native-packages/wsdb/cpp/src/merkle_tree/node_store/content_addressed_cache.hpp similarity index 99% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/content_addressed_cache.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/node_store/content_addressed_cache.hpp index 40ef06125ed4..dc8dcecc97a9 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/content_addressed_cache.hpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/node_store/content_addressed_cache.hpp @@ -5,9 +5,9 @@ // ===================== #pragma once -#include "./tree_meta.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "barretenberg/crypto/merkle_tree/tree_meta.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" #include "barretenberg/numeric/uint256/uint256.hpp" diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/content_addressed_cache.test.cpp b/native-packages/wsdb/cpp/src/merkle_tree/node_store/content_addressed_cache.test.cpp similarity index 98% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/content_addressed_cache.test.cpp rename to native-packages/wsdb/cpp/src/merkle_tree/node_store/content_addressed_cache.test.cpp index e690308530a0..c5f127a224d6 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/content_addressed_cache.test.cpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/node_store/content_addressed_cache.test.cpp @@ -1,8 +1,8 @@ -#include "barretenberg/crypto/merkle_tree/node_store/content_addressed_cache.hpp" +#include "merkle_tree/node_store/content_addressed_cache.hpp" #include "barretenberg/common/test.hpp" -#include "barretenberg/crypto/merkle_tree/fixtures.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" +#include "merkle_tree/fixtures.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/tree_meta.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" #include #include diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp b/native-packages/wsdb/cpp/src/merkle_tree/nullifier_tree/nullifier_leaf.hpp similarity index 100% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_leaf.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/nullifier_tree/nullifier_leaf.hpp diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp b/native-packages/wsdb/cpp/src/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp similarity index 98% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp index b13f0f7500d8..298408fa4bfd 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/nullifier_tree/nullifier_memory_tree.hpp @@ -5,8 +5,8 @@ // ===================== #pragma once -#include "../hash.hpp" -#include "../memory_tree.hpp" +#include "barretenberg/crypto/merkle_tree/hash.hpp" +#include "barretenberg/crypto/merkle_tree/memory_tree.hpp" #include "barretenberg/common/assert.hpp" #include "barretenberg/crypto/merkle_tree/hash_path.hpp" #include "nullifier_leaf.hpp" diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp b/native-packages/wsdb/cpp/src/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp similarity index 100% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp rename to native-packages/wsdb/cpp/src/merkle_tree/nullifier_tree/nullifier_memory_tree.test.cpp diff --git a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp b/native-packages/wsdb/cpp/src/merkle_tree/test_fixtures.hpp similarity index 98% rename from barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp rename to native-packages/wsdb/cpp/src/merkle_tree/test_fixtures.hpp index e7a56a52848f..0ca0aee29356 100644 --- a/barretenberg/cpp/src/barretenberg/crypto/merkle_tree/test_fixtures.hpp +++ b/native-packages/wsdb/cpp/src/merkle_tree/test_fixtures.hpp @@ -1,7 +1,7 @@ #pragma once -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/crypto/merkle_tree/signal.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" diff --git a/native-packages/wsdb/cpp/src/merkle_tree/tree_db_stats.hpp b/native-packages/wsdb/cpp/src/merkle_tree/tree_db_stats.hpp new file mode 100644 index 000000000000..84b300e24a64 --- /dev/null +++ b/native-packages/wsdb/cpp/src/merkle_tree/tree_db_stats.hpp @@ -0,0 +1,123 @@ +#pragma once + +// Persistent-tree storage statistics and the merkle responses that carry them. +// These live in the wsdb package (not barretenberg's generic merkle core) +// because only the persistent, lmdb-backed trees report stats — the in-memory +// merkle tree has none. DBStats itself is owned by lmdblib (it is what lmdb +// reports). + +#include "barretenberg/crypto/merkle_tree/tree_meta.hpp" +#include "barretenberg/serialize/msgpack.hpp" +#include "lmdblib/types.hpp" +#include +#include + +namespace bb::crypto::merkle_tree { + +using DBStats = bb::lmdblib::DBStats; + +struct TreeDBStats { + uint64_t mapSize; + uint64_t physicalFileSize; + DBStats blocksDBStats; + DBStats nodesDBStats; + DBStats leafPreimagesDBStats; + DBStats leafIndicesDBStats; + DBStats blockIndicesDBStats; + + TreeDBStats() = default; + TreeDBStats(uint64_t mapSize, uint64_t physicalFileSize) + : mapSize(mapSize), physicalFileSize(physicalFileSize) {} + TreeDBStats(uint64_t mapSize, uint64_t physicalFileSize, + const DBStats &blockStats, const DBStats &nodesStats, + const DBStats &leafPreimagesDBStats, + const DBStats &leafIndicesStats, const DBStats &blockIndicesStats) + : mapSize(mapSize), physicalFileSize(physicalFileSize), + blocksDBStats(blockStats), nodesDBStats(nodesStats), + leafPreimagesDBStats(leafPreimagesDBStats), + leafIndicesDBStats(leafIndicesStats), + blockIndicesDBStats(blockIndicesStats) {} + TreeDBStats(const TreeDBStats &other) = default; + TreeDBStats(TreeDBStats &&other) noexcept { *this = std::move(other); } + + ~TreeDBStats() = default; + + SERIALIZATION_FIELDS(mapSize, physicalFileSize, blocksDBStats, nodesDBStats, + leafPreimagesDBStats, leafIndicesDBStats, + blockIndicesDBStats) + + bool operator==(const TreeDBStats &other) const { + return mapSize == other.mapSize && + physicalFileSize == other.physicalFileSize && + blocksDBStats == other.blocksDBStats && + nodesDBStats == other.nodesDBStats && + leafPreimagesDBStats == other.leafPreimagesDBStats && + leafIndicesDBStats == other.leafIndicesDBStats && + blockIndicesDBStats == other.blockIndicesDBStats; + } + + TreeDBStats &operator=(TreeDBStats &&other) noexcept { + if (this != &other) { + mapSize = other.mapSize; + physicalFileSize = other.physicalFileSize; + blocksDBStats = std::move(other.blocksDBStats); + nodesDBStats = std::move(other.nodesDBStats); + leafPreimagesDBStats = std::move(other.leafPreimagesDBStats); + leafIndicesDBStats = std::move(other.leafIndicesDBStats); + blockIndicesDBStats = std::move(other.blockIndicesDBStats); + } + return *this; + } + + TreeDBStats &operator=(const TreeDBStats &other) = default; + + friend std::ostream &operator<<(std::ostream &os, const TreeDBStats &stats) { + os << "Map Size: " << stats.mapSize + << ", Physical File Size: " << stats.physicalFileSize << " Blocks DB " + << stats.blocksDBStats << ", Nodes DB " << stats.nodesDBStats + << ", Leaf Pre-images DB " << stats.leafPreimagesDBStats + << ", Leaf Indices DB " << stats.leafIndicesDBStats + << ", Block Indices DB " << stats.blockIndicesDBStats; + return os; + } +}; + +struct CommitResponse { + TreeMeta meta; + TreeDBStats stats; + + CommitResponse() = default; + ~CommitResponse() = default; + CommitResponse(const CommitResponse &other) = default; + CommitResponse(CommitResponse &&other) noexcept = default; + CommitResponse &operator=(const CommitResponse &other) = default; + CommitResponse &operator=(CommitResponse &&other) noexcept = default; +}; + +struct UnwindResponse { + TreeMeta meta; + TreeDBStats stats; + + UnwindResponse() = default; + ~UnwindResponse() = default; + UnwindResponse(const UnwindResponse &other) = default; + UnwindResponse(UnwindResponse &&other) noexcept = default; + UnwindResponse &operator=(const UnwindResponse &other) = default; + UnwindResponse &operator=(UnwindResponse &&other) noexcept = default; +}; + +struct RemoveHistoricResponse { + TreeMeta meta; + TreeDBStats stats; + + RemoveHistoricResponse() = default; + ~RemoveHistoricResponse() = default; + RemoveHistoricResponse(const RemoveHistoricResponse &other) = default; + RemoveHistoricResponse(RemoveHistoricResponse &&other) noexcept = default; + RemoveHistoricResponse & + operator=(const RemoveHistoricResponse &other) = default; + RemoveHistoricResponse & + operator=(RemoveHistoricResponse &&other) noexcept = default; +}; + +} // namespace bb::crypto::merkle_tree diff --git a/barretenberg/cpp/src/barretenberg/world_state/fork.hpp b/native-packages/wsdb/cpp/src/world_state/fork.hpp similarity index 76% rename from barretenberg/cpp/src/barretenberg/world_state/fork.hpp rename to native-packages/wsdb/cpp/src/world_state/fork.hpp index 63134bb2b83d..4f1dd6f6f37e 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/fork.hpp +++ b/native-packages/wsdb/cpp/src/world_state/fork.hpp @@ -1,11 +1,11 @@ #pragma once #include "barretenberg/aztec/aztec_hash_policy.hpp" -#include "barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp" -#include "barretenberg/world_state/tree_with_store.hpp" -#include "barretenberg/world_state/types.hpp" +#include "merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" +#include "merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp" +#include "merkle_tree/node_store/cached_content_addressed_tree_store.hpp" +#include "world_state/tree_with_store.hpp" +#include "world_state/types.hpp" #include #include diff --git a/barretenberg/cpp/src/barretenberg/world_state/tree_with_store.hpp b/native-packages/wsdb/cpp/src/world_state/tree_with_store.hpp similarity index 100% rename from barretenberg/cpp/src/barretenberg/world_state/tree_with_store.hpp rename to native-packages/wsdb/cpp/src/world_state/tree_with_store.hpp diff --git a/barretenberg/cpp/src/barretenberg/world_state/types.cpp b/native-packages/wsdb/cpp/src/world_state/types.cpp similarity index 79% rename from barretenberg/cpp/src/barretenberg/world_state/types.cpp rename to native-packages/wsdb/cpp/src/world_state/types.cpp index 3be022ed8297..6172d0f68cd4 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/types.cpp +++ b/native-packages/wsdb/cpp/src/world_state/types.cpp @@ -1,4 +1,4 @@ -#include "barretenberg/world_state/types.hpp" +#include "world_state/types.hpp" // getMerkleTreeName now lives in crypto/merkle_tree/merkle_tree_id.cpp and is re-exported into // bb::world_state via the `using namespace bb::crypto::merkle_tree` in types.hpp. diff --git a/barretenberg/cpp/src/barretenberg/world_state/types.hpp b/native-packages/wsdb/cpp/src/world_state/types.hpp similarity index 97% rename from barretenberg/cpp/src/barretenberg/world_state/types.hpp rename to native-packages/wsdb/cpp/src/world_state/types.hpp index 2aa4688e6af0..0d04b31c0a14 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/types.hpp +++ b/native-packages/wsdb/cpp/src/world_state/types.hpp @@ -5,10 +5,11 @@ #include #include -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/tree_db_stats.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" #include "barretenberg/crypto/merkle_tree/merkle_tree_id.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" +#include "barretenberg/crypto/merkle_tree/tree_meta.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" #include "barretenberg/serialize/msgpack.hpp" diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp b/native-packages/wsdb/cpp/src/world_state/world_state.cpp similarity index 98% rename from barretenberg/cpp/src/barretenberg/world_state/world_state.cpp rename to native-packages/wsdb/cpp/src/world_state/world_state.cpp index 883fefbd0e9b..d1a6e4d880ea 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.cpp +++ b/native-packages/wsdb/cpp/src/world_state/world_state.cpp @@ -1,19 +1,19 @@ -#include "barretenberg/world_state/world_state.hpp" +#include "world_state/world_state.hpp" #include "barretenberg/aztec/aztec_constants.hpp" -#include "barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" +#include "merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" #include "barretenberg/crypto/merkle_tree/hash.hpp" #include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "barretenberg/crypto/merkle_tree/tree_meta.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/crypto/merkle_tree/signal.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/lmdblib/lmdb_helpers.hpp" -#include "barretenberg/world_state/fork.hpp" -#include "barretenberg/world_state/tree_with_store.hpp" -#include "barretenberg/world_state/types.hpp" -#include "barretenberg/world_state/world_state_stores.hpp" +#include "lmdblib/lmdb_helpers.hpp" +#include "world_state/fork.hpp" +#include "world_state/tree_with_store.hpp" +#include "world_state/types.hpp" +#include "world_state/world_state_stores.hpp" #include #include #include diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp b/native-packages/wsdb/cpp/src/world_state/world_state.hpp similarity index 97% rename from barretenberg/cpp/src/barretenberg/world_state/world_state.hpp rename to native-packages/wsdb/cpp/src/world_state/world_state.hpp index 93a04f614ccc..3baa086fd7c8 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.hpp +++ b/native-packages/wsdb/cpp/src/world_state/world_state.hpp @@ -1,23 +1,24 @@ #pragma once #include "barretenberg/common/thread_pool.hpp" -#include "barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" +#include "merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp" +#include "merkle_tree/tree_db_stats.hpp" #include "barretenberg/crypto/merkle_tree/hash.hpp" #include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" +#include "merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/node_store/cached_content_addressed_tree_store.hpp" +#include "barretenberg/crypto/merkle_tree/tree_meta.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/crypto/merkle_tree/signal.hpp" #include "barretenberg/crypto/merkle_tree/types.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/lmdblib/lmdb_environment.hpp" +#include "lmdblib/lmdb_environment.hpp" #include "barretenberg/serialize/msgpack.hpp" -#include "barretenberg/world_state/fork.hpp" -#include "barretenberg/world_state/tree_with_store.hpp" -#include "barretenberg/world_state/types.hpp" -#include "barretenberg/world_state/world_state_stores.hpp" +#include "world_state/fork.hpp" +#include "world_state/tree_with_store.hpp" +#include "world_state/types.hpp" +#include "world_state/world_state_stores.hpp" #include #include #include diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp b/native-packages/wsdb/cpp/src/world_state/world_state.test.cpp similarity index 99% rename from barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp rename to native-packages/wsdb/cpp/src/world_state/world_state.test.cpp index 1d24634e9de0..cd957b40d517 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state.test.cpp +++ b/native-packages/wsdb/cpp/src/world_state/world_state.test.cpp @@ -1,12 +1,12 @@ -#include "barretenberg/world_state/world_state.hpp" +#include "world_state/world_state.hpp" #include "barretenberg/aztec/aztec_constants.hpp" -#include "barretenberg/crypto/merkle_tree/fixtures.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" +#include "merkle_tree/fixtures.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/tree_meta.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" #include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/world_state/fork.hpp" -#include "barretenberg/world_state/types.hpp" +#include "world_state/fork.hpp" +#include "world_state/types.hpp" #include #include #include diff --git a/barretenberg/cpp/src/barretenberg/world_state/world_state_stores.hpp b/native-packages/wsdb/cpp/src/world_state/world_state_stores.hpp similarity index 95% rename from barretenberg/cpp/src/barretenberg/world_state/world_state_stores.hpp rename to native-packages/wsdb/cpp/src/world_state/world_state_stores.hpp index 31c924d0a0cb..d2dba1478685 100644 --- a/barretenberg/cpp/src/barretenberg/world_state/world_state_stores.hpp +++ b/native-packages/wsdb/cpp/src/world_state/world_state_stores.hpp @@ -1,6 +1,6 @@ #pragma once -#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp" +#include "merkle_tree/lmdb_store/lmdb_tree_store.hpp" #include #include diff --git a/barretenberg/cpp/src/barretenberg/wsdb/cli.cpp b/native-packages/wsdb/cpp/src/wsdb/cli.cpp similarity index 95% rename from barretenberg/cpp/src/barretenberg/wsdb/cli.cpp rename to native-packages/wsdb/cpp/src/wsdb/cli.cpp index 2cf90fb38bcb..569a3dde7201 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb/cli.cpp +++ b/native-packages/wsdb/cpp/src/wsdb/cli.cpp @@ -1,10 +1,10 @@ -#include "barretenberg/wsdb/cli.hpp" +#include "wsdb/cli.hpp" #include "barretenberg/common/log.hpp" #include "barretenberg/common/throw_or_abort.hpp" #include "barretenberg/serialize/msgpack.hpp" -#include "barretenberg/world_state/world_state.hpp" -#include "barretenberg/wsdb/generated/wsdb_ipc_server.hpp" -#include "barretenberg/wsdb/wsdb_ipc_server.hpp" +#include "world_state/world_state.hpp" +#include "wsdb/generated/wsdb_ipc_server.hpp" +#include "wsdb/wsdb_ipc_server.hpp" #include "barretenberg/bb/deps/cli11.hpp" #include diff --git a/barretenberg/cpp/src/barretenberg/wsdb/cli.hpp b/native-packages/wsdb/cpp/src/wsdb/cli.hpp similarity index 100% rename from barretenberg/cpp/src/barretenberg/wsdb/cli.hpp rename to native-packages/wsdb/cpp/src/wsdb/cli.hpp diff --git a/barretenberg/cpp/src/barretenberg/wsdb/main.cpp b/native-packages/wsdb/cpp/src/wsdb/main.cpp similarity index 71% rename from barretenberg/cpp/src/barretenberg/wsdb/main.cpp rename to native-packages/wsdb/cpp/src/wsdb/main.cpp index 8857922c678a..78926f8b2385 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb/main.cpp +++ b/native-packages/wsdb/cpp/src/wsdb/main.cpp @@ -1,4 +1,4 @@ -#include "barretenberg/wsdb/cli.hpp" +#include "wsdb/cli.hpp" int main(int argc, char* argv[]) { diff --git a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/memory_merkle_db.test.cpp b/native-packages/wsdb/cpp/src/wsdb/memory_merkle_db.test.cpp similarity index 98% rename from barretenberg/cpp/src/barretenberg/vm2/simulation/lib/memory_merkle_db.test.cpp rename to native-packages/wsdb/cpp/src/wsdb/memory_merkle_db.test.cpp index 94c34a0cb28a..6e43cbf66635 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/simulation/lib/memory_merkle_db.test.cpp +++ b/native-packages/wsdb/cpp/src/wsdb/memory_merkle_db.test.cpp @@ -8,11 +8,11 @@ #include #include "barretenberg/aztec/aztec_constants.hpp" -#include "barretenberg/crypto/merkle_tree/fixtures.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "merkle_tree/fixtures.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/crypto/merkle_tree/merkle_tree_id.hpp" #include "barretenberg/vm2/common/field.hpp" -#include "barretenberg/world_state/world_state.hpp" +#include "world_state/world_state.hpp" // This test is the canonical-fidelity gate for MemoryMerkleDB. It drives an ephemeral, file-backed // world_state::WorldState and an in-memory MemoryMerkleDB through an identical sequence of operations diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_handlers.cpp b/native-packages/wsdb/cpp/src/wsdb/wsdb_handlers.cpp similarity index 99% rename from barretenberg/cpp/src/barretenberg/wsdb/wsdb_handlers.cpp rename to native-packages/wsdb/cpp/src/wsdb/wsdb_handlers.cpp index 91a6da868191..a1fc3de90d2e 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_handlers.cpp +++ b/native-packages/wsdb/cpp/src/wsdb/wsdb_handlers.cpp @@ -15,12 +15,13 @@ * into respond.ok(...) or a thrown exception into respond.error(...). Wire <-> * domain conversion happens inside the lambda via wsdb_wire_convert.hpp. */ -#include "barretenberg/wsdb/wsdb_handlers.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "wsdb/wsdb_handlers.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" +#include "merkle_tree/tree_db_stats.hpp" #include "barretenberg/crypto/merkle_tree/response.hpp" -#include "barretenberg/world_state/world_state.hpp" -#include "barretenberg/wsdb/wsdb_schedule.hpp" -#include "barretenberg/wsdb/wsdb_wire_convert.hpp" +#include "world_state/world_state.hpp" +#include "wsdb/wsdb_schedule.hpp" +#include "wsdb/wsdb_wire_convert.hpp" #include #include diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_handlers.hpp b/native-packages/wsdb/cpp/src/wsdb/wsdb_handlers.hpp similarity index 98% rename from barretenberg/cpp/src/barretenberg/wsdb/wsdb_handlers.hpp rename to native-packages/wsdb/cpp/src/wsdb/wsdb_handlers.hpp index 886cd0297611..d9dd9d517b32 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_handlers.hpp +++ b/native-packages/wsdb/cpp/src/wsdb/wsdb_handlers.hpp @@ -14,9 +14,10 @@ * wsdb_schedule.hpp) and the response is produced by respond.ok()/error() when * the work completes. Definitions live in wsdb_handlers.cpp. */ -#include "barretenberg/wsdb/generated/wsdb_dispatch.hpp" // Responder -#include "barretenberg/wsdb/generated/wsdb_types.hpp" -#include "barretenberg/wsdb/wsdb_request.hpp" +#include "merkle_tree/tree_db_stats.hpp" +#include "wsdb/generated/wsdb_dispatch.hpp" // Responder +#include "wsdb/generated/wsdb_types.hpp" +#include "wsdb/wsdb_request.hpp" namespace bb::wsdb { diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_server.cpp b/native-packages/wsdb/cpp/src/wsdb/wsdb_ipc_server.cpp similarity index 96% rename from barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_server.cpp rename to native-packages/wsdb/cpp/src/wsdb/wsdb_ipc_server.cpp index ca98eb60e12e..8a7518711ac0 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_server.cpp +++ b/native-packages/wsdb/cpp/src/wsdb/wsdb_ipc_server.cpp @@ -1,13 +1,13 @@ -#include "barretenberg/wsdb/wsdb_ipc_server.hpp" +#include "wsdb/wsdb_ipc_server.hpp" #include "barretenberg/common/log.hpp" #include "barretenberg/common/thread_pool.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" #include "barretenberg/serialize/msgpack.hpp" -#include "barretenberg/world_state/world_state.hpp" -#include "barretenberg/wsdb/generated/wsdb_ipc_server.hpp" -#include "barretenberg/wsdb/wsdb_handlers.hpp" -#include "barretenberg/wsdb/wsdb_request.hpp" -#include "barretenberg/wsdb/wsdb_scheduler.hpp" +#include "world_state/world_state.hpp" +#include "wsdb/generated/wsdb_ipc_server.hpp" +#include "wsdb/wsdb_handlers.hpp" +#include "wsdb/wsdb_request.hpp" +#include "wsdb/wsdb_scheduler.hpp" #include "ipc_runtime/ipc_server.hpp" #include "ipc_runtime/serve_helper.hpp" #include "ipc_runtime/signal_handlers.hpp" diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_server.hpp b/native-packages/wsdb/cpp/src/wsdb/wsdb_ipc_server.hpp similarity index 100% rename from barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_server.hpp rename to native-packages/wsdb/cpp/src/wsdb/wsdb_ipc_server.hpp diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_request.hpp b/native-packages/wsdb/cpp/src/wsdb/wsdb_request.hpp similarity index 93% rename from barretenberg/cpp/src/barretenberg/wsdb/wsdb_request.hpp rename to native-packages/wsdb/cpp/src/wsdb/wsdb_request.hpp index 530ab629fae3..08dc5a5b6ba2 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_request.hpp +++ b/native-packages/wsdb/cpp/src/wsdb/wsdb_request.hpp @@ -8,7 +8,7 @@ * work to (so reads run concurrently and writes are serialized per fork — see * wsdb_schedule.hpp / wsdb_scheduler.hpp). */ -#include "barretenberg/world_state/world_state.hpp" +#include "world_state/world_state.hpp" namespace bb::wsdb { diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_schedule.hpp b/native-packages/wsdb/cpp/src/wsdb/wsdb_schedule.hpp similarity index 92% rename from barretenberg/cpp/src/barretenberg/wsdb/wsdb_schedule.hpp rename to native-packages/wsdb/cpp/src/wsdb/wsdb_schedule.hpp index aa6e0d9e63a0..77989d2e2f78 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_schedule.hpp +++ b/native-packages/wsdb/cpp/src/wsdb/wsdb_schedule.hpp @@ -12,9 +12,9 @@ * bodies stay free of threading and error-frame boilerplate. */ -#include "barretenberg/wsdb/generated/wsdb_dispatch.hpp" // Responder -#include "barretenberg/wsdb/wsdb_request.hpp" -#include "barretenberg/wsdb/wsdb_scheduler.hpp" +#include "wsdb/generated/wsdb_dispatch.hpp" // Responder +#include "wsdb/wsdb_request.hpp" +#include "wsdb/wsdb_scheduler.hpp" #include #include diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_scheduler.hpp b/native-packages/wsdb/cpp/src/wsdb/wsdb_scheduler.hpp similarity index 100% rename from barretenberg/cpp/src/barretenberg/wsdb/wsdb_scheduler.hpp rename to native-packages/wsdb/cpp/src/wsdb/wsdb_scheduler.hpp diff --git a/native-packages/wsdb/cpp/src/wsdb/wsdb_wire_convert.hpp b/native-packages/wsdb/cpp/src/wsdb/wsdb_wire_convert.hpp new file mode 100644 index 000000000000..6f6074ed32fe --- /dev/null +++ b/native-packages/wsdb/cpp/src/wsdb/wsdb_wire_convert.hpp @@ -0,0 +1,175 @@ +#pragma once +/** + * @file wsdb_wire_convert.hpp + * @brief Server-side wire <-> domain conversion helpers for the aztec-wsdb + * service. + * + * The converters that depend only on the generic merkle-tree vocabulary live + * in a local wsdb_wire_convert_client.hpp, bound to this package's generated + * wire types (bb's wsdb client keeps its own copy bound to bb's generated + * types; the two can't share one header because each side codegens its own + * wsdb_types.hpp). This header pulls those in and adds the server-only + * converters that touch world_state aggregates (state references, DB stats, + * tree/world-state meta), which require world_state and the persistent merkle + * storage. + */ +#include "barretenberg/crypto/merkle_tree/response.hpp" +#include "merkle_tree/tree_db_stats.hpp" +#include "barretenberg/crypto/merkle_tree/tree_meta.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" +#include "lmdblib/types.hpp" +#include "world_state/types.hpp" +#include "world_state/world_state.hpp" +#include "wsdb/wsdb_wire_convert_client.hpp" + +namespace bb::wsdb { + +inline std::vector +state_reference_to_wire(const world_state::StateReference &d) { + std::vector r; + r.reserve(d.size()); + for (const auto &[tree_id, tree_ref] : d) { + r.push_back({.treeId = tree_id_to_wire(tree_id), + .root = fr_to_wire(tree_ref.first), + .size = tree_ref.second}); + } + return r; +} + +inline world_state::StateReference +state_reference_from_wire(const std::vector &w) { + world_state::StateReference r; + r.reserve(w.size()); + for (const auto &entry : w) { + r.emplace( + tree_id_from_wire(entry.treeId), + world_state::TreeStateReference{fr_from_wire(entry.root), entry.size}); + } + return r; +} + +inline wire::DBStats db_stats_to_wire(const bb::lmdblib::DBStats &d) { + return {.name = d.name, + .numDataItems = d.numDataItems, + .totalUsedSize = d.totalUsedSize}; +} + +inline bb::lmdblib::DBStats db_stats_from_wire(const wire::DBStats &w) { + return bb::lmdblib::DBStats(w.name, w.numDataItems, w.totalUsedSize); +} + +inline wire::TreeDBStats +tree_db_stats_to_wire(const bb::crypto::merkle_tree::TreeDBStats &d) { + return {.mapSize = d.mapSize, + .physicalFileSize = d.physicalFileSize, + .blocksDBStats = db_stats_to_wire(d.blocksDBStats), + .nodesDBStats = db_stats_to_wire(d.nodesDBStats), + .leafPreimagesDBStats = db_stats_to_wire(d.leafPreimagesDBStats), + .leafIndicesDBStats = db_stats_to_wire(d.leafIndicesDBStats), + .blockIndicesDBStats = db_stats_to_wire(d.blockIndicesDBStats)}; +} + +inline bb::crypto::merkle_tree::TreeDBStats +tree_db_stats_from_wire(const wire::TreeDBStats &w) { + return {w.mapSize, + w.physicalFileSize, + db_stats_from_wire(w.blocksDBStats), + db_stats_from_wire(w.nodesDBStats), + db_stats_from_wire(w.leafPreimagesDBStats), + db_stats_from_wire(w.leafIndicesDBStats), + db_stats_from_wire(w.blockIndicesDBStats)}; +} + +inline wire::TreeMeta +tree_meta_to_wire(const bb::crypto::merkle_tree::TreeMeta &d) { + return {.name = d.name, + .depth = d.depth, + .size = d.size, + .committedSize = d.committedSize, + .root = fr_to_wire(d.root), + .initialSize = d.initialSize, + .initialRoot = fr_to_wire(d.initialRoot), + .oldestHistoricBlock = d.oldestHistoricBlock, + .unfinalizedBlockHeight = d.unfinalizedBlockHeight, + .finalizedBlockHeight = d.finalizedBlockHeight}; +} + +inline bb::crypto::merkle_tree::TreeMeta +tree_meta_from_wire(const wire::TreeMeta &w) { + return {w.name, + w.depth, + w.size, + w.committedSize, + fr_from_wire(w.root), + w.initialSize, + fr_from_wire(w.initialRoot), + w.oldestHistoricBlock, + w.unfinalizedBlockHeight, + w.finalizedBlockHeight}; +} + +inline wire::WorldStateDBStats +world_state_db_stats_to_wire(const bb::world_state::WorldStateDBStats &d) { + return {.noteHashTreeStats = tree_db_stats_to_wire(d.noteHashTreeStats), + .messageTreeStats = tree_db_stats_to_wire(d.messageTreeStats), + .archiveTreeStats = tree_db_stats_to_wire(d.archiveTreeStats), + .publicDataTreeStats = tree_db_stats_to_wire(d.publicDataTreeStats), + .nullifierTreeStats = tree_db_stats_to_wire(d.nullifierTreeStats)}; +} + +inline bb::world_state::WorldStateDBStats +world_state_db_stats_from_wire(const wire::WorldStateDBStats &w) { + return {tree_db_stats_from_wire(w.noteHashTreeStats), + tree_db_stats_from_wire(w.messageTreeStats), + tree_db_stats_from_wire(w.archiveTreeStats), + tree_db_stats_from_wire(w.publicDataTreeStats), + tree_db_stats_from_wire(w.nullifierTreeStats)}; +} + +inline wire::WorldStateMeta +world_state_meta_to_wire(const bb::world_state::WorldStateMeta &d) { + return {.noteHashTreeMeta = tree_meta_to_wire(d.noteHashTreeMeta), + .messageTreeMeta = tree_meta_to_wire(d.messageTreeMeta), + .archiveTreeMeta = tree_meta_to_wire(d.archiveTreeMeta), + .publicDataTreeMeta = tree_meta_to_wire(d.publicDataTreeMeta), + .nullifierTreeMeta = tree_meta_to_wire(d.nullifierTreeMeta)}; +} + +inline bb::world_state::WorldStateMeta +world_state_meta_from_wire(const wire::WorldStateMeta &w) { + return {tree_meta_from_wire(w.noteHashTreeMeta), + tree_meta_from_wire(w.messageTreeMeta), + tree_meta_from_wire(w.archiveTreeMeta), + tree_meta_from_wire(w.publicDataTreeMeta), + tree_meta_from_wire(w.nullifierTreeMeta)}; +} + +inline wire::WorldStateStatusSummary world_state_status_summary_to_wire( + const bb::world_state::WorldStateStatusSummary &d) { + return {.unfinalizedBlockNumber = d.unfinalizedBlockNumber, + .finalizedBlockNumber = d.finalizedBlockNumber, + .oldestHistoricalBlock = d.oldestHistoricalBlock, + .treesAreSynched = d.treesAreSynched}; +} + +inline bb::world_state::WorldStateStatusSummary +world_state_status_summary_from_wire(const wire::WorldStateStatusSummary &w) { + return {w.unfinalizedBlockNumber, w.finalizedBlockNumber, + w.oldestHistoricalBlock, w.treesAreSynched}; +} + +inline wire::WorldStateStatusFull world_state_status_full_to_wire( + const bb::world_state::WorldStateStatusFull &d) { + return {.summary = world_state_status_summary_to_wire(d.summary), + .dbStats = world_state_db_stats_to_wire(d.dbStats), + .meta = world_state_meta_to_wire(d.meta)}; +} + +inline bb::world_state::WorldStateStatusFull +world_state_status_full_from_wire(const wire::WorldStateStatusFull &w) { + return {world_state_status_summary_from_wire(w.summary), + world_state_db_stats_from_wire(w.dbStats), + world_state_meta_from_wire(w.meta)}; +} + +} // namespace bb::wsdb diff --git a/native-packages/wsdb/cpp/src/wsdb/wsdb_wire_convert_client.hpp b/native-packages/wsdb/cpp/src/wsdb/wsdb_wire_convert_client.hpp new file mode 100644 index 000000000000..2e275af1afa1 --- /dev/null +++ b/native-packages/wsdb/cpp/src/wsdb/wsdb_wire_convert_client.hpp @@ -0,0 +1,362 @@ +#pragma once +/** + * @file wsdb_wire_convert.hpp + * @brief Client-side wire <-> domain conversion helpers for the aztec-wsdb service. + * + * These converters depend only on the generic merkle-tree vocabulary + * (crypto/merkle_tree) plus the generated wsdb wire types, so they can be + * compiled into bb's wsdb client (consumed by the AVM simulator) without + * pulling in world_state or the persistent merkle storage. The server-side + * converters that touch world_state aggregates (state references, DB stats, + * tree/world-state meta) live alongside the server in native-packages/wsdb. + */ +#include "barretenberg/crypto/merkle_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/merkle_tree_id.hpp" +#include "barretenberg/crypto/merkle_tree/response.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" +#include "barretenberg/ecc/curves/bn254/fr.hpp" +#include "wsdb/generated/wsdb_types.hpp" + +namespace bb::wsdb { + +inline Fr fr_to_wire(const bb::fr& d) +{ + Fr r{}; + bb::fr::serialize_to_buffer(d, r.data()); + return r; +} + +inline bb::fr fr_from_wire(const Fr& w) +{ + return bb::fr::serialize_from_buffer(w.data()); +} + +inline BlockHeaderHash block_header_hash_to_wire(const bb::fr& d) +{ + BlockHeaderHash r{}; + bb::fr::serialize_to_buffer(d, r.data()); + return r; +} + +inline bb::fr block_header_hash_from_wire(const BlockHeaderHash& w) +{ + return bb::fr::serialize_from_buffer(w.data()); +} + +inline PublicDataSlot public_data_slot_to_wire(const bb::fr& d) +{ + PublicDataSlot r{}; + bb::fr::serialize_to_buffer(d, r.data()); + return r; +} + +inline bb::fr public_data_slot_from_wire(const PublicDataSlot& w) +{ + return bb::fr::serialize_from_buffer(w.data()); +} + +inline PublicDataValue public_data_value_to_wire(const bb::fr& d) +{ + PublicDataValue r{}; + bb::fr::serialize_to_buffer(d, r.data()); + return r; +} + +inline bb::fr public_data_value_from_wire(const PublicDataValue& w) +{ + return bb::fr::serialize_from_buffer(w.data()); +} + +inline Nullifier nullifier_to_wire(const bb::fr& d) +{ + Nullifier r{}; + bb::fr::serialize_to_buffer(d, r.data()); + return r; +} + +inline bb::fr nullifier_from_wire(const Nullifier& w) +{ + return bb::fr::serialize_from_buffer(w.data()); +} + +inline std::vector fr_vec_to_wire(const std::vector& d) +{ + std::vector r; + r.reserve(d.size()); + for (const auto& x : d) { + r.push_back(fr_to_wire(x)); + } + return r; +} + +inline std::vector fr_vec_from_wire(const std::vector& w) +{ + std::vector r; + r.reserve(w.size()); + for (const auto& x : w) { + r.push_back(fr_from_wire(x)); + } + return r; +} + +inline wire::WorldStateRevision revision_to_wire(const crypto::merkle_tree::WorldStateRevision& d) +{ + return wire::WorldStateRevision{ + .forkId = d.forkId, + .blockNumber = d.blockNumber, + .includeUncommitted = d.includeUncommitted, + }; +} + +inline crypto::merkle_tree::WorldStateRevision revision_from_wire(const wire::WorldStateRevision& w) +{ + return crypto::merkle_tree::WorldStateRevision{ + .forkId = w.forkId, + .blockNumber = w.blockNumber, + .includeUncommitted = w.includeUncommitted, + }; +} + +inline MerkleTreeId tree_id_to_wire(crypto::merkle_tree::MerkleTreeId d) +{ + return static_cast(d); +} + +inline crypto::merkle_tree::MerkleTreeId tree_id_from_wire(MerkleTreeId w) +{ + return static_cast(w); +} + +inline wire::PublicDataLeafValue public_data_leaf_to_wire(const crypto::merkle_tree::PublicDataLeafValue& d) +{ + return { .slot = public_data_slot_to_wire(d.slot), .value = public_data_value_to_wire(d.value) }; +} + +inline crypto::merkle_tree::PublicDataLeafValue public_data_leaf_from_wire(const wire::PublicDataLeafValue& w) +{ + return { public_data_slot_from_wire(w.slot), public_data_value_from_wire(w.value) }; +} + +inline std::vector public_data_leaf_vec_to_wire( + const std::vector& d) +{ + std::vector r; + r.reserve(d.size()); + for (const auto& x : d) { + r.push_back(public_data_leaf_to_wire(x)); + } + return r; +} + +inline std::vector public_data_leaf_vec_from_wire( + const std::vector& w) +{ + std::vector r; + r.reserve(w.size()); + for (const auto& x : w) { + r.push_back(public_data_leaf_from_wire(x)); + } + return r; +} + +inline wire::NullifierLeafValue nullifier_leaf_to_wire(const crypto::merkle_tree::NullifierLeafValue& d) +{ + return { .nullifier = nullifier_to_wire(d.nullifier) }; +} + +inline crypto::merkle_tree::NullifierLeafValue nullifier_leaf_from_wire(const wire::NullifierLeafValue& w) +{ + return { nullifier_from_wire(w.nullifier) }; +} + +inline std::vector nullifier_leaf_vec_to_wire( + const std::vector& d) +{ + std::vector r; + r.reserve(d.size()); + for (const auto& x : d) { + r.push_back(nullifier_leaf_to_wire(x)); + } + return r; +} + +inline std::vector nullifier_leaf_vec_from_wire( + const std::vector& w) +{ + std::vector r; + r.reserve(w.size()); + for (const auto& x : w) { + r.push_back(nullifier_leaf_from_wire(x)); + } + return r; +} + +inline wire::IndexedPublicDataLeafValue indexed_public_data_leaf_to_wire( + const crypto::merkle_tree::IndexedLeaf& d) +{ + return { .leaf = public_data_leaf_to_wire(d.leaf), .nextIndex = d.nextIndex, .nextKey = fr_to_wire(d.nextKey) }; +} + +inline crypto::merkle_tree::IndexedLeaf indexed_public_data_leaf_from_wire( + const wire::IndexedPublicDataLeafValue& w) +{ + return { public_data_leaf_from_wire(w.leaf), w.nextIndex, fr_from_wire(w.nextKey) }; +} + +inline wire::IndexedNullifierLeafValue indexed_nullifier_leaf_to_wire( + const crypto::merkle_tree::IndexedLeaf& d) +{ + return { .leaf = nullifier_leaf_to_wire(d.leaf), .nextIndex = d.nextIndex, .nextKey = fr_to_wire(d.nextKey) }; +} + +inline crypto::merkle_tree::IndexedLeaf indexed_nullifier_leaf_from_wire( + const wire::IndexedNullifierLeafValue& w) +{ + return { nullifier_leaf_from_wire(w.leaf), w.nextIndex, fr_from_wire(w.nextKey) }; +} + +inline wire::PublicDataLeafUpdateWitnessData public_data_witness_to_wire( + const crypto::merkle_tree::LeafUpdateWitnessData& d) +{ + return { .leaf = indexed_public_data_leaf_to_wire(d.leaf), .index = d.index, .path = fr_vec_to_wire(d.path) }; +} + +inline crypto::merkle_tree::LeafUpdateWitnessData +public_data_witness_from_wire(const wire::PublicDataLeafUpdateWitnessData& w) +{ + return { indexed_public_data_leaf_from_wire(w.leaf), w.index, fr_vec_from_wire(w.path) }; +} + +inline wire::NullifierLeafUpdateWitnessData nullifier_witness_to_wire( + const crypto::merkle_tree::LeafUpdateWitnessData& d) +{ + return { .leaf = indexed_nullifier_leaf_to_wire(d.leaf), .index = d.index, .path = fr_vec_to_wire(d.path) }; +} + +inline crypto::merkle_tree::LeafUpdateWitnessData nullifier_witness_from_wire( + const wire::NullifierLeafUpdateWitnessData& w) +{ + return { indexed_nullifier_leaf_from_wire(w.leaf), w.index, fr_vec_from_wire(w.path) }; +} + +template +inline std::vector vec_to_wire(const std::vector& d, Fn fn) +{ + std::vector r; + r.reserve(d.size()); + for (const auto& x : d) { + r.push_back(fn(x)); + } + return r; +} + +template +inline std::vector vec_from_wire(const std::vector& w, Fn fn) +{ + std::vector r; + r.reserve(w.size()); + for (const auto& x : w) { + r.push_back(fn(x)); + } + return r; +} + +inline wire::BatchInsertionResultPublicData batch_public_data_to_wire( + const crypto::merkle_tree::BatchInsertionResult& d) +{ + std::vector sorted; + sorted.reserve(d.sorted_leaves.size()); + for (const auto& [leaf, index] : d.sorted_leaves) { + sorted.push_back({ .leaf = public_data_leaf_to_wire(leaf), .index = index }); + } + return { .lowLeafWitnessData = vec_to_wire(d.low_leaf_witness_data, + public_data_witness_to_wire), + .sortedLeaves = std::move(sorted), + .subtreePath = fr_vec_to_wire(d.subtree_path) }; +} + +inline crypto::merkle_tree::BatchInsertionResult batch_public_data_from_wire( + const wire::BatchInsertionResultPublicData& w) +{ + crypto::merkle_tree::BatchInsertionResult r; + r.low_leaf_witness_data = + vec_from_wire>( + w.lowLeafWitnessData, public_data_witness_from_wire); + r.sorted_leaves.reserve(w.sortedLeaves.size()); + for (const auto& x : w.sortedLeaves) { + r.sorted_leaves.emplace_back(public_data_leaf_from_wire(x.leaf), x.index); + } + r.subtree_path = fr_vec_from_wire(w.subtreePath); + return r; +} + +inline wire::BatchInsertionResultNullifier batch_nullifier_to_wire( + const crypto::merkle_tree::BatchInsertionResult& d) +{ + std::vector sorted; + sorted.reserve(d.sorted_leaves.size()); + for (const auto& [leaf, index] : d.sorted_leaves) { + sorted.push_back({ .leaf = nullifier_leaf_to_wire(leaf), .index = index }); + } + return { .lowLeafWitnessData = + vec_to_wire(d.low_leaf_witness_data, nullifier_witness_to_wire), + .sortedLeaves = std::move(sorted), + .subtreePath = fr_vec_to_wire(d.subtree_path) }; +} + +inline crypto::merkle_tree::BatchInsertionResult batch_nullifier_from_wire( + const wire::BatchInsertionResultNullifier& w) +{ + crypto::merkle_tree::BatchInsertionResult r; + r.low_leaf_witness_data = + vec_from_wire>( + w.lowLeafWitnessData, nullifier_witness_from_wire); + r.sorted_leaves.reserve(w.sortedLeaves.size()); + for (const auto& x : w.sortedLeaves) { + r.sorted_leaves.emplace_back(nullifier_leaf_from_wire(x.leaf), x.index); + } + r.subtree_path = fr_vec_from_wire(w.subtreePath); + return r; +} + +inline wire::SequentialInsertionResultPublicData sequential_public_data_to_wire( + const crypto::merkle_tree::SequentialInsertionResult& d) +{ + return { .lowLeafWitnessData = vec_to_wire(d.low_leaf_witness_data, + public_data_witness_to_wire), + .insertionWitnessData = vec_to_wire(d.insertion_witness_data, + public_data_witness_to_wire) }; +} + +inline crypto::merkle_tree::SequentialInsertionResult +sequential_public_data_from_wire(const wire::SequentialInsertionResultPublicData& w) +{ + return { .low_leaf_witness_data = + vec_from_wire>( + w.lowLeafWitnessData, public_data_witness_from_wire), + .insertion_witness_data = + vec_from_wire>( + w.insertionWitnessData, public_data_witness_from_wire) }; +} + +inline wire::SequentialInsertionResultNullifier sequential_nullifier_to_wire( + const crypto::merkle_tree::SequentialInsertionResult& d) +{ + return { .lowLeafWitnessData = + vec_to_wire(d.low_leaf_witness_data, nullifier_witness_to_wire), + .insertionWitnessData = vec_to_wire(d.insertion_witness_data, + nullifier_witness_to_wire) }; +} + +inline crypto::merkle_tree::SequentialInsertionResult +sequential_nullifier_from_wire(const wire::SequentialInsertionResultNullifier& w) +{ + return { .low_leaf_witness_data = + vec_from_wire>( + w.lowLeafWitnessData, nullifier_witness_from_wire), + .insertion_witness_data = + vec_from_wire>( + w.insertionWitnessData, nullifier_witness_from_wire) }; +} + +} // namespace bb::wsdb diff --git a/wsdb/package.json b/native-packages/wsdb/package.json similarity index 84% rename from wsdb/package.json rename to native-packages/wsdb/package.json index d2a88a1534f9..cb84a9543890 100644 --- a/wsdb/package.json +++ b/native-packages/wsdb/package.json @@ -11,6 +11,6 @@ "ts/packages/*" ], "resolutions": { - "@aztec/ipc-runtime": "portal:../ipc-runtime/ts" + "@aztec/ipc-runtime": "portal:../../ipc-runtime/ts" } } diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_schema.jsonc b/native-packages/wsdb/wsdb_schema.jsonc similarity index 100% rename from barretenberg/cpp/src/barretenberg/wsdb/wsdb_schema.jsonc rename to native-packages/wsdb/wsdb_schema.jsonc diff --git a/wsdb/yarn.lock b/native-packages/wsdb/yarn.lock similarity index 98% rename from wsdb/yarn.lock rename to native-packages/wsdb/yarn.lock index 22dfff672f55..d0f2cc8d4e7d 100644 --- a/wsdb/yarn.lock +++ b/native-packages/wsdb/yarn.lock @@ -5,9 +5,9 @@ __metadata: version: 8 cacheKey: 10c0 -"@aztec/ipc-runtime@portal:../ipc-runtime/ts::locator=%40aztec%2Fwsdb-packages%40workspace%3A.": +"@aztec/ipc-runtime@portal:../../ipc-runtime/ts::locator=%40aztec%2Fwsdb-packages%40workspace%3A.": version: 0.0.0-use.local - resolution: "@aztec/ipc-runtime@portal:../ipc-runtime/ts::locator=%40aztec%2Fwsdb-packages%40workspace%3A." + resolution: "@aztec/ipc-runtime@portal:../../ipc-runtime/ts::locator=%40aztec%2Fwsdb-packages%40workspace%3A." languageName: node linkType: soft diff --git a/release-image/Dockerfile.base.dockerignore b/release-image/Dockerfile.base.dockerignore index cc831e424879..f5765b793415 100644 --- a/release-image/Dockerfile.base.dockerignore +++ b/release-image/Dockerfile.base.dockerignore @@ -4,8 +4,10 @@ !/barretenberg/ts/bb-avm-sim/package.json !/barretenberg/ts/bb-avm-sim/packages/*/package.json !/noir/packages/*/package.json -!/wsdb/ts/package.json -!/wsdb/ts/packages/*/package.json +!/native-packages/wsdb/ts/package.json +!/native-packages/wsdb/ts/packages/*/package.json +!/native-packages/kvdb/ts/package.json +!/native-packages/kvdb/ts/packages/*/package.json !/ipc-runtime/ts/package.json !/yarn-project/package.json !/yarn-project/yarn.lock diff --git a/release-image/Dockerfile.dockerignore b/release-image/Dockerfile.dockerignore index 067ce2fb1c59..921556978686 100644 --- a/release-image/Dockerfile.dockerignore +++ b/release-image/Dockerfile.dockerignore @@ -11,10 +11,14 @@ !/barretenberg/ts/bb-avm-sim/packages/ !/ipc-runtime/ts/dest/ !/ipc-runtime/ts/package.json -!/wsdb/ts/dest/ -!/wsdb/ts/build/ -!/wsdb/ts/package.json -!/wsdb/ts/packages/ +!/native-packages/wsdb/ts/dest/ +!/native-packages/wsdb/ts/build/ +!/native-packages/wsdb/ts/package.json +!/native-packages/wsdb/ts/packages/ +!/native-packages/kvdb/ts/dest/ +!/native-packages/kvdb/ts/build/ +!/native-packages/kvdb/ts/package.json +!/native-packages/kvdb/ts/packages/ !/noir/noir-repo/target/release/nargo !/noir/noir-repo/target/release/acvm !/noir/packages/ diff --git a/release-image/bootstrap.sh b/release-image/bootstrap.sh index 8ca6698f3c4d..eec3361e8fad 100755 --- a/release-image/bootstrap.sh +++ b/release-image/bootstrap.sh @@ -3,7 +3,8 @@ source $(git rev-parse --show-toplevel)/ci3/source_bootstrap hash=$(hash_str \ $(cache_content_hash ^release-image/Dockerfile ^release-image/Dockerfile.base.dockerignore ^release-image/Dockerfile.dockerignore ^build-images/src/Dockerfile ^ipc-runtime/ts/package.json ^yarn-project/yarn.lock) \ - $(../wsdb/bootstrap.sh hash) \ + $(../native-packages/kvdb/bootstrap.sh hash) \ + $(../native-packages/wsdb/bootstrap.sh hash) \ $(../barretenberg/ts/bootstrap.sh hash)) function prepare_crs { diff --git a/wsdb/.rebuild_patterns b/wsdb/.rebuild_patterns deleted file mode 100644 index 1652d3c01a63..000000000000 --- a/wsdb/.rebuild_patterns +++ /dev/null @@ -1,6 +0,0 @@ -^wsdb/bootstrap\.sh$ -^wsdb/package\.json$ -^wsdb/\.rebuild_patterns$ -^ipc-codegen/src/.*\.ts$ -^ipc-codegen/templates/ -^barretenberg/cpp/src/barretenberg/wsdb/wsdb_schema\.json$ diff --git a/yarn-project/native/package.json b/yarn-project/native/package.json index 4c0fb4df0f58..8583fcb6fc9b 100644 --- a/yarn-project/native/package.json +++ b/yarn-project/native/package.json @@ -17,6 +17,7 @@ "dependencies": { "@aztec/bb.js": "workspace:^", "@aztec/foundation": "workspace:^", + "@aztec/kvdb": "0.1.0", "msgpackr": "^1.11.2" }, "devDependencies": { diff --git a/yarn-project/native/src/native_module.ts b/yarn-project/native/src/native_module.ts index 25d477906e35..2bf94be62ee3 100644 --- a/yarn-project/native/src/native_module.ts +++ b/yarn-project/native/src/native_module.ts @@ -1,4 +1,4 @@ -import { findNapiBinary } from '@aztec/bb.js'; +import { findNapiBinary } from '@aztec/kvdb'; import { createRequire } from 'module'; diff --git a/yarn-project/package.json b/yarn-project/package.json index d58fce70531a..9b60f2d9b55c 100644 --- a/yarn-project/package.json +++ b/yarn-project/package.json @@ -92,11 +92,16 @@ "@aztec/bb-avm-sim-linux-x64": "portal:../barretenberg/ts/bb-avm-sim/packages/bb-avm-sim-linux-x64", "@aztec/bb.js": "portal:../barretenberg/ts/bb.js", "@aztec/ipc-runtime": "portal:../ipc-runtime/ts", - "@aztec/wsdb": "portal:../wsdb/ts", - "@aztec/wsdb-darwin-arm64": "portal:../wsdb/ts/packages/wsdb-darwin-arm64", - "@aztec/wsdb-darwin-x64": "portal:../wsdb/ts/packages/wsdb-darwin-x64", - "@aztec/wsdb-linux-arm64": "portal:../wsdb/ts/packages/wsdb-linux-arm64", - "@aztec/wsdb-linux-x64": "portal:../wsdb/ts/packages/wsdb-linux-x64", + "@aztec/wsdb": "portal:../native-packages/wsdb/ts", + "@aztec/wsdb-darwin-arm64": "portal:../native-packages/wsdb/ts/packages/wsdb-darwin-arm64", + "@aztec/wsdb-darwin-x64": "portal:../native-packages/wsdb/ts/packages/wsdb-darwin-x64", + "@aztec/wsdb-linux-arm64": "portal:../native-packages/wsdb/ts/packages/wsdb-linux-arm64", + "@aztec/wsdb-linux-x64": "portal:../native-packages/wsdb/ts/packages/wsdb-linux-x64", + "@aztec/kvdb": "portal:../native-packages/kvdb/ts", + "@aztec/kvdb-darwin-arm64": "portal:../native-packages/kvdb/ts/packages/kvdb-darwin-arm64", + "@aztec/kvdb-darwin-x64": "portal:../native-packages/kvdb/ts/packages/kvdb-darwin-x64", + "@aztec/kvdb-linux-arm64": "portal:../native-packages/kvdb/ts/packages/kvdb-linux-arm64", + "@aztec/kvdb-linux-x64": "portal:../native-packages/kvdb/ts/packages/kvdb-linux-x64", "@aztec/noir-acvm_js": "portal:../noir/packages/acvm_js", "@aztec/noir-types": "portal:../noir/packages/types", "@aztec/noir-noirc_abi": "portal:../noir/packages/noirc_abi", diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 4439e960a681..430f4a021648 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -1567,6 +1567,55 @@ __metadata: languageName: unknown linkType: soft +"@aztec/kvdb-darwin-arm64@portal:../native-packages/kvdb/ts/packages/kvdb-darwin-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": + version: 0.0.0-use.local + resolution: "@aztec/kvdb-darwin-arm64@portal:../native-packages/kvdb/ts/packages/kvdb-darwin-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A." + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: soft + +"@aztec/kvdb-darwin-x64@portal:../native-packages/kvdb/ts/packages/kvdb-darwin-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": + version: 0.0.0-use.local + resolution: "@aztec/kvdb-darwin-x64@portal:../native-packages/kvdb/ts/packages/kvdb-darwin-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A." + conditions: os=darwin & cpu=x64 + languageName: node + linkType: soft + +"@aztec/kvdb-linux-arm64@portal:../native-packages/kvdb/ts/packages/kvdb-linux-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": + version: 0.0.0-use.local + resolution: "@aztec/kvdb-linux-arm64@portal:../native-packages/kvdb/ts/packages/kvdb-linux-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A." + conditions: os=linux & cpu=arm64 + languageName: node + linkType: soft + +"@aztec/kvdb-linux-x64@portal:../native-packages/kvdb/ts/packages/kvdb-linux-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": + version: 0.0.0-use.local + resolution: "@aztec/kvdb-linux-x64@portal:../native-packages/kvdb/ts/packages/kvdb-linux-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A." + conditions: os=linux & cpu=x64 + languageName: node + linkType: soft + +"@aztec/kvdb@portal:../native-packages/kvdb/ts::locator=%40aztec%2Faztec3-packages%40workspace%3A.": + version: 0.0.0-use.local + resolution: "@aztec/kvdb@portal:../native-packages/kvdb/ts::locator=%40aztec%2Faztec3-packages%40workspace%3A." + dependencies: + "@aztec/kvdb-darwin-arm64": "npm:0.1.0" + "@aztec/kvdb-darwin-x64": "npm:0.1.0" + "@aztec/kvdb-linux-arm64": "npm:0.1.0" + "@aztec/kvdb-linux-x64": "npm:0.1.0" + tslib: "npm:^2.4.0" + dependenciesMeta: + "@aztec/kvdb-darwin-arm64": + optional: true + "@aztec/kvdb-darwin-x64": + optional: true + "@aztec/kvdb-linux-arm64": + optional: true + "@aztec/kvdb-linux-x64": + optional: true + languageName: node + linkType: soft + "@aztec/l1-artifacts@workspace:^, @aztec/l1-artifacts@workspace:l1-artifacts": version: 0.0.0-use.local resolution: "@aztec/l1-artifacts@workspace:l1-artifacts" @@ -1585,6 +1634,7 @@ __metadata: dependencies: "@aztec/bb.js": "workspace:^" "@aztec/foundation": "workspace:^" + "@aztec/kvdb": "npm:0.1.0" "@jest/globals": "npm:^30.0.0" "@types/jest": "npm:^30.0.0" "@types/node": "npm:^22.15.17" @@ -2411,37 +2461,37 @@ __metadata: languageName: unknown linkType: soft -"@aztec/wsdb-darwin-arm64@portal:../wsdb/ts/packages/wsdb-darwin-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": +"@aztec/wsdb-darwin-arm64@portal:../native-packages/wsdb/ts/packages/wsdb-darwin-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": version: 0.0.0-use.local - resolution: "@aztec/wsdb-darwin-arm64@portal:../wsdb/ts/packages/wsdb-darwin-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A." + resolution: "@aztec/wsdb-darwin-arm64@portal:../native-packages/wsdb/ts/packages/wsdb-darwin-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A." conditions: os=darwin & cpu=arm64 languageName: node linkType: soft -"@aztec/wsdb-darwin-x64@portal:../wsdb/ts/packages/wsdb-darwin-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": +"@aztec/wsdb-darwin-x64@portal:../native-packages/wsdb/ts/packages/wsdb-darwin-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": version: 0.0.0-use.local - resolution: "@aztec/wsdb-darwin-x64@portal:../wsdb/ts/packages/wsdb-darwin-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A." + resolution: "@aztec/wsdb-darwin-x64@portal:../native-packages/wsdb/ts/packages/wsdb-darwin-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A." conditions: os=darwin & cpu=x64 languageName: node linkType: soft -"@aztec/wsdb-linux-arm64@portal:../wsdb/ts/packages/wsdb-linux-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": +"@aztec/wsdb-linux-arm64@portal:../native-packages/wsdb/ts/packages/wsdb-linux-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": version: 0.0.0-use.local - resolution: "@aztec/wsdb-linux-arm64@portal:../wsdb/ts/packages/wsdb-linux-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A." + resolution: "@aztec/wsdb-linux-arm64@portal:../native-packages/wsdb/ts/packages/wsdb-linux-arm64::locator=%40aztec%2Faztec3-packages%40workspace%3A." conditions: os=linux & cpu=arm64 languageName: node linkType: soft -"@aztec/wsdb-linux-x64@portal:../wsdb/ts/packages/wsdb-linux-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": +"@aztec/wsdb-linux-x64@portal:../native-packages/wsdb/ts/packages/wsdb-linux-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A.": version: 0.0.0-use.local - resolution: "@aztec/wsdb-linux-x64@portal:../wsdb/ts/packages/wsdb-linux-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A." + resolution: "@aztec/wsdb-linux-x64@portal:../native-packages/wsdb/ts/packages/wsdb-linux-x64::locator=%40aztec%2Faztec3-packages%40workspace%3A." conditions: os=linux & cpu=x64 languageName: node linkType: soft -"@aztec/wsdb@portal:../wsdb/ts::locator=%40aztec%2Faztec3-packages%40workspace%3A.": +"@aztec/wsdb@portal:../native-packages/wsdb/ts::locator=%40aztec%2Faztec3-packages%40workspace%3A.": version: 0.0.0-use.local - resolution: "@aztec/wsdb@portal:../wsdb/ts::locator=%40aztec%2Faztec3-packages%40workspace%3A." + resolution: "@aztec/wsdb@portal:../native-packages/wsdb/ts::locator=%40aztec%2Faztec3-packages%40workspace%3A." dependencies: "@aztec/ipc-runtime": "@aztec/ipc-runtime" "@aztec/wsdb-darwin-arm64": "npm:0.1.0"