Skip to content

Commit d83a271

Browse files
charlielyecharlielye
committed
refactor: make lmdblib + kvdb fully barretenberg-free
Addresses review: native-packages should not depend on barretenberg. - lmdblib: zero barretenberg includes/links. Uses msgpack-c directly (MSGPACK_DEFINE_MAP — byte-identical wire to bb's SERIALIZATION_FIELDS); serialise_key for field-like keys is now a template (preserves the on-disk byte layout) so no uint256 type is needed; owns DBStats; local format/THROW helpers. Builds against lmdb + msgpack-c only. - kvdb: zero barretenberg includes/links. Split nodejs_module — only the LMDBStore NAPI lives here; the msgpack_client wrappers stay in barretenberg for bb.js's SHM transport. Vendors the small message header/dispatcher it needs; links only lmdblib + lmdb + node-addon-api. - bb.js: reverted — it builds and ships its own nodejs_module.node (msgpack_client only) from barretenberg exactly as before. No native-packages dependency. - DBStats is owned by lmdblib (lmdb stats). TreeDBStats and the stats-bearing merkle responses move to the wsdb package (only the persistent trees report stats); barretenberg's generic merkle core carries no stats vocabulary. - Build wiring: lmdblib and kvdb no longer depend on bb-cpp-native; yarn-project depends on kvdb; bb-ts reverts to no kvdb dependency.
1 parent 81b5b8e commit d83a271

58 files changed

Lines changed: 1493 additions & 418 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Makefile‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -219,9 +219,7 @@ bb-cpp-release-dir: bb-cpp-native bb-cpp-cross bb-cpp-wasm bb-cpp-wasm-threads
219219
bb-cpp-full: bb-cpp bb-cpp-gcc bb-cpp-fuzzing bb-cpp-asan bb-cpp-smt bb-cpp-cross-arm64-macos bb-cpp-cross-arm64-ios bb-cpp-cross-arm64-android
220220

221221
# BB TypeScript - TypeScript bindings
222-
# bb-ts depends on kvdb: bb.js ships the kvdb NAPI addon (nodejs_module.node) for
223-
# its SHM transport (see barretenberg/ts/bb.js/scripts/copy_native.sh).
224-
bb-ts: bb-cpp-wasm bb-cpp-wasm-threads bb-cpp-native ipc-runtime kvdb
222+
bb-ts: bb-cpp-wasm bb-cpp-wasm-threads bb-cpp-native ipc-runtime
225223
$(call build,$@,barretenberg/ts,build_bb_js)
226224

227225
# Copies the cross-compiles into bb.js.
@@ -326,11 +324,13 @@ ipc-runtime-cross: ipc-runtime ipc-runtime-cross-arm64-linux ipc-runtime-cross-a
326324
# Native packages (lmdblib, kvdb, wsdb)
327325
#==============================================================================
328326

327+
# lmdblib and kvdb are barretenberg-free: they build against their own deps
328+
# (lmdb, msgpack-c, node-addon-api) only, never bb.
329329
.PHONY: lmdblib kvdb
330-
lmdblib: bb-cpp-native
330+
lmdblib:
331331
$(call build,$@,native-packages/lmdblib)
332332

333-
kvdb: lmdblib bb-cpp-native
333+
kvdb: lmdblib
334334
$(call build,$@,native-packages/kvdb)
335335

336336
wsdb: ipc-codegen ipc-runtime bb-cpp-native lmdblib
@@ -402,7 +402,7 @@ l1-contracts-tests: l1-contracts-verifier
402402
# Yarn Project - TypeScript monorepo with all TS packages
403403
#==============================================================================
404404

405-
yarn-project: bb-ts noir-projects l1-contracts wsdb bb-avm-sim
405+
yarn-project: bb-ts noir-projects l1-contracts wsdb kvdb bb-avm-sim
406406
$(call build,$@,yarn-project)
407407

408408
yarn-project-tests: yarn-project

‎barretenberg/cpp/bootstrap.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ function build_native_objects {
120120
if ! cache_exists barretenberg-$native_preset-$hash.zst; then
121121
cmake --preset "$native_preset"
122122
targets=$(cmake --build --preset "$native_preset" --target help | awk -F: '$1 ~ /(_objects|_tests|_bench|_gen|.a)$/ && $1 !~ /^cmake_/{print $1}' | tr '\n' ' ')
123-
cmake --build --preset "$native_preset" --target $targets
123+
cmake --build --preset "$native_preset" --target $targets nodejs_module
124124
fi
125125
}
126126

‎barretenberg/cpp/src/CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,9 @@ if(NOT FUZZING AND NOT WASM AND NOT BB_LITE)
124124
add_subdirectory(barretenberg/vm2_wsdb)
125125
add_subdirectory(barretenberg/cdb)
126126
add_subdirectory(barretenberg/avm)
127+
# nodejs_module hosts only the msgpack_client NAPI wrappers now (bb.js's SHM
128+
# transport). The lmdb_store kv-store NAPI moved to native-packages/kvdb.
129+
add_subdirectory(barretenberg/nodejs_module)
127130
endif()
128131

129132
# Pull in ipc-runtime as a C++ dependency. Provides the `ipc_runtime`

‎barretenberg/cpp/src/barretenberg/crypto/merkle_tree/db_stats.hpp‎

Lines changed: 0 additions & 59 deletions
This file was deleted.

‎barretenberg/cpp/src/barretenberg/crypto/merkle_tree/response.hpp‎

Lines changed: 0 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -217,42 +217,6 @@ struct GetLowIndexedLeafResponse {
217217
}
218218
};
219219

220-
struct CommitResponse {
221-
TreeMeta meta;
222-
TreeDBStats stats;
223-
224-
CommitResponse() = default;
225-
~CommitResponse() = default;
226-
CommitResponse(const CommitResponse& other) = default;
227-
CommitResponse(CommitResponse&& other) noexcept = default;
228-
CommitResponse& operator=(const CommitResponse& other) = default;
229-
CommitResponse& operator=(CommitResponse&& other) noexcept = default;
230-
};
231-
232-
struct UnwindResponse {
233-
TreeMeta meta;
234-
TreeDBStats stats;
235-
236-
UnwindResponse() = default;
237-
~UnwindResponse() = default;
238-
UnwindResponse(const UnwindResponse& other) = default;
239-
UnwindResponse(UnwindResponse&& other) noexcept = default;
240-
UnwindResponse& operator=(const UnwindResponse& other) = default;
241-
UnwindResponse& operator=(UnwindResponse&& other) noexcept = default;
242-
};
243-
244-
struct RemoveHistoricResponse {
245-
TreeMeta meta;
246-
TreeDBStats stats;
247-
248-
RemoveHistoricResponse() = default;
249-
~RemoveHistoricResponse() = default;
250-
RemoveHistoricResponse(const RemoveHistoricResponse& other) = default;
251-
RemoveHistoricResponse(RemoveHistoricResponse&& other) noexcept = default;
252-
RemoveHistoricResponse& operator=(const RemoveHistoricResponse& other) = default;
253-
RemoveHistoricResponse& operator=(RemoveHistoricResponse&& other) noexcept = default;
254-
};
255-
256220
template <typename ResponseType> struct TypedResponse {
257221
ResponseType inner;
258222
bool success{ true };

‎barretenberg/cpp/src/barretenberg/crypto/merkle_tree/types.hpp‎

Lines changed: 0 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66

77
#pragma once
88

9-
#include "barretenberg/crypto/merkle_tree/db_stats.hpp"
109
#include "barretenberg/ecc/curves/bn254/fr.hpp"
1110
#include <cstdint>
1211
#include <optional>
@@ -62,81 +61,4 @@ const std::string LEAF_PREIMAGES_DB = "leaf preimages";
6261
const std::string LEAF_INDICES_DB = "leaf indices";
6362
const std::string BLOCK_INDICES_DB = "block indices";
6463

65-
struct TreeDBStats {
66-
uint64_t mapSize;
67-
uint64_t physicalFileSize;
68-
DBStats blocksDBStats;
69-
DBStats nodesDBStats;
70-
DBStats leafPreimagesDBStats;
71-
DBStats leafIndicesDBStats;
72-
DBStats blockIndicesDBStats;
73-
74-
TreeDBStats() = default;
75-
TreeDBStats(uint64_t mapSize, uint64_t physicalFileSize)
76-
: mapSize(mapSize)
77-
, physicalFileSize(physicalFileSize)
78-
{}
79-
TreeDBStats(uint64_t mapSize,
80-
uint64_t physicalFileSize,
81-
const DBStats& blockStats,
82-
const DBStats& nodesStats,
83-
const DBStats& leafPreimagesDBStats,
84-
const DBStats& leafIndicesStats,
85-
const DBStats& blockIndicesStats)
86-
: mapSize(mapSize)
87-
, physicalFileSize(physicalFileSize)
88-
, blocksDBStats(blockStats)
89-
, nodesDBStats(nodesStats)
90-
, leafPreimagesDBStats(leafPreimagesDBStats)
91-
, leafIndicesDBStats(leafIndicesStats)
92-
, blockIndicesDBStats(blockIndicesStats)
93-
{}
94-
TreeDBStats(const TreeDBStats& other) = default;
95-
TreeDBStats(TreeDBStats&& other) noexcept { *this = std::move(other); }
96-
97-
~TreeDBStats() = default;
98-
99-
SERIALIZATION_FIELDS(mapSize,
100-
physicalFileSize,
101-
blocksDBStats,
102-
nodesDBStats,
103-
leafPreimagesDBStats,
104-
leafIndicesDBStats,
105-
blockIndicesDBStats)
106-
107-
bool operator==(const TreeDBStats& other) const
108-
{
109-
return mapSize == other.mapSize && physicalFileSize == other.physicalFileSize &&
110-
blocksDBStats == other.blocksDBStats && nodesDBStats == other.nodesDBStats &&
111-
leafPreimagesDBStats == other.leafPreimagesDBStats && leafIndicesDBStats == other.leafIndicesDBStats &&
112-
blockIndicesDBStats == other.blockIndicesDBStats;
113-
}
114-
115-
TreeDBStats& operator=(TreeDBStats&& other) noexcept
116-
{
117-
if (this != &other) {
118-
mapSize = other.mapSize;
119-
physicalFileSize = other.physicalFileSize;
120-
blocksDBStats = std::move(other.blocksDBStats);
121-
nodesDBStats = std::move(other.nodesDBStats);
122-
leafPreimagesDBStats = std::move(other.leafPreimagesDBStats);
123-
leafIndicesDBStats = std::move(other.leafIndicesDBStats);
124-
blockIndicesDBStats = std::move(other.blockIndicesDBStats);
125-
}
126-
return *this;
127-
}
128-
129-
TreeDBStats& operator=(const TreeDBStats& other) = default;
130-
131-
friend std::ostream& operator<<(std::ostream& os, const TreeDBStats& stats)
132-
{
133-
os << "Map Size: " << stats.mapSize << ", Physical File Size: " << stats.physicalFileSize << " Blocks DB "
134-
<< stats.blocksDBStats << ", Nodes DB " << stats.nodesDBStats << ", Leaf Pre-images DB "
135-
<< stats.leafPreimagesDBStats << ", Leaf Indices DB " << stats.leafIndicesDBStats << ", Block Indices DB "
136-
<< stats.blockIndicesDBStats;
137-
return os;
138-
}
139-
};
140-
141-
std::ostream& operator<<(std::ostream& os, const TreeDBStats& stats);
14264
} // namespace bb::crypto::merkle_tree

‎barretenberg/cpp/src/barretenberg/nodejs_module/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
.yarn

‎barretenberg/cpp/src/barretenberg/nodejs_module/.yarnrc.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
nodeLinker: node-modules
2+
3+
npmMinimalAgeGate: 7d
4+
5+
npmPreapprovedPackages: []

‎barretenberg/cpp/src/barretenberg/nodejs_module/CMakeLists.txt‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# see https://nodejs.org/dist/latest/docs/api/n-api.html#node-api-version-matrix
2+
add_definitions(-DNAPI_VERSION=9)
3+
4+
file(GLOB_RECURSE SOURCE_FILES *.cpp)
5+
file(GLOB_RECURSE HEADER_FILES *.hpp *.tcc)
6+
7+
execute_process(
8+
COMMAND yarn --immutable
9+
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
10+
)
11+
12+
execute_process(
13+
COMMAND node -p "require('node-addon-api').include"
14+
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
15+
OUTPUT_VARIABLE NODE_ADDON_API_DIR
16+
)
17+
18+
execute_process(
19+
COMMAND node -p "require('node-api-headers').include_dir"
20+
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
21+
OUTPUT_VARIABLE NODE_API_HEADERS_DIR
22+
)
23+
24+
string(REGEX REPLACE "[\r\n\"]" "" NODE_ADDON_API_DIR ${NODE_ADDON_API_DIR})
25+
string(REGEX REPLACE "[\r\n\"]" "" NODE_API_HEADERS_DIR ${NODE_API_HEADERS_DIR})
26+
27+
add_library(nodejs_module SHARED ${SOURCE_FILES})
28+
set_target_properties(nodejs_module PROPERTIES PREFIX "" SUFFIX ".node")
29+
target_include_directories(nodejs_module PRIVATE ${NODE_API_HEADERS_DIR} ${NODE_ADDON_API_DIR})
30+
target_link_libraries(nodejs_module PRIVATE ipc ipc_runtime)
31+
32+
# On macOS, Node.js N-API symbols are provided by the runtime, not at link time
33+
if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
34+
target_link_options(nodejs_module PRIVATE "-undefined" "dynamic_lookup")
35+
endif()

‎barretenberg/cpp/src/barretenberg/nodejs_module/init_module.cpp‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#include "barretenberg/nodejs_module/msgpack_client/msgpack_client_async.hpp"
2+
#include "barretenberg/nodejs_module/msgpack_client/msgpack_client_wrapper.hpp"
3+
#include "napi.h"
4+
5+
Napi::Object Init(Napi::Env env, Napi::Object exports)
6+
{
7+
exports.Set(Napi::String::New(env, "MsgpackClient"),
8+
bb::nodejs::msgpack_client::MsgpackClientWrapper::get_class(env));
9+
exports.Set(Napi::String::New(env, "MsgpackClientAsync"),
10+
bb::nodejs::msgpack_client::MsgpackClientAsync::get_class(env));
11+
return exports;
12+
}
13+
14+
// NOLINTNEXTLINE
15+
NODE_API_MODULE(addon, Init)

0 commit comments

Comments
 (0)