Skip to content

Commit 1ecdf78

Browse files
charlielyecharlielye
committed
refactor: decouple wsdb from barretenberg headers
wsdb compiles with zero barretenberg headers — its own FieldElement (32-byte canonical) + a poseidon2 c_bind are the only barretenberg link. Forks the bb-free common utils, moves merkle_tree Signal into wsdb, keeps lmdb keys backward-compatible (little-endian uint256), switches lmdblib to upstream msgpack + FetchContent (bb-free), and wires the lmdblib/wsdb C++ tests into the CI test engine. Squashed from the decouple branch (10 commits).
1 parent 2218d58 commit 1ecdf78

111 files changed

Lines changed: 2596 additions & 650 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Makefile‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ endef
5555

5656
# Fast bootstrap.
5757
fast: release-image barretenberg boxes playground docs aztec-up \
58-
bb-tests l1-contracts-tests yarn-project-tests boxes-tests playground-tests aztec-up-tests docs-tests noir-protocol-circuits-tests release-image-tests spartan claude-tests ipc-codegen-tests
58+
bb-tests l1-contracts-tests yarn-project-tests boxes-tests playground-tests aztec-up-tests docs-tests noir-protocol-circuits-tests release-image-tests spartan claude-tests ipc-codegen-tests lmdblib-tests wsdb-tests
5959

6060
# Full bootstrap.
6161
full: fast bb-full-tests bb-cpp-full yarn-project-benches
@@ -326,7 +326,7 @@ ipc-runtime-cross: ipc-runtime ipc-runtime-cross-arm64-linux ipc-runtime-cross-a
326326

327327
# lmdblib and kvdb are barretenberg-free: they build against their own deps
328328
# (lmdb, msgpack-c, node-addon-api) only, never bb.
329-
.PHONY: lmdblib kvdb
329+
.PHONY: lmdblib kvdb lmdblib-tests wsdb-tests
330330
lmdblib:
331331
$(call build,$@,native-packages/lmdblib)
332332

@@ -336,6 +336,15 @@ kvdb: lmdblib
336336
wsdb: ipc-codegen ipc-runtime bb-cpp-native lmdblib
337337
$(call build,$@,native-packages/wsdb)
338338

339+
# Native-package C++ tests (self-contained gtest binaries). kvdb has no C++ tests
340+
# (its NAPI is exercised by yarn-project's kv-store tests). wsdb_tests use no bb
341+
# headers; the bb-header parity/equivalence target is manual (WSDB_BUILD_BB_TESTS).
342+
lmdblib-tests: lmdblib
343+
$(call test,$@,native-packages/lmdblib)
344+
345+
wsdb-tests: wsdb
346+
$(call test,$@,native-packages/wsdb)
347+
339348
#==============================================================================
340349
# .claude tooling
341350
#==============================================================================

‎barretenberg/cpp/src/barretenberg/crypto/poseidon2/poseidon2_capi.cpp‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#include "barretenberg/crypto/poseidon2/poseidon2.hpp"
2+
#include "barretenberg/crypto/poseidon2/poseidon2_params.hpp"
3+
#include "barretenberg/ecc/curves/bn254/fr.hpp"
4+
#include <cstddef>
5+
#include <cstdint>
6+
#include <vector>
7+
8+
// Flat C ABI over Poseidon2, for in-process consumers that must hash at high
9+
// frequency but want no dependency on barretenberg's C++ headers, field type,
10+
// stdlib, or build flags (e.g. the standalone world-state DB service). Field
11+
// elements cross as 32 canonical bytes (barretenberg's field serialization);
12+
// the montgomery form stays private to barretenberg.
13+
namespace {
14+
using FF = bb::fr;
15+
using Poseidon2Bn254 = bb::crypto::Poseidon2<bb::crypto::Poseidon2Bn254ScalarFieldParams>;
16+
} // namespace
17+
18+
extern "C" {
19+
20+
// Hash `n` field elements. `elems` is n*32 canonical bytes; `out` receives 32.
21+
void bb_poseidon2_hash(const uint8_t* elems, size_t n, uint8_t* out)
22+
{
23+
std::vector<FF> inputs;
24+
inputs.reserve(n);
25+
for (size_t i = 0; i < n; ++i) {
26+
inputs.push_back(FF::serialize_from_buffer(elems + (i * 32)));
27+
}
28+
FF::serialize_to_buffer(Poseidon2Bn254::hash(inputs), out);
29+
}
30+
31+
// Hash { separator, lhs, rhs } — matches Poseidon2HashPolicy::hash_pair_with_separator,
32+
// the domain-separated internal-node hash used by the Aztec merkle trees.
33+
void bb_poseidon2_hash_pair_with_separator(uint64_t separator, const uint8_t* lhs, const uint8_t* rhs, uint8_t* out)
34+
{
35+
std::vector<FF> inputs{ FF(separator), FF::serialize_from_buffer(lhs), FF::serialize_from_buffer(rhs) };
36+
FF::serialize_to_buffer(Poseidon2Bn254::hash(inputs), out);
37+
}
38+
39+
} // extern "C"

‎native-packages/kvdb/cpp/CMakeLists.txt‎

Lines changed: 7 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -30,36 +30,20 @@ get_filename_component(REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../.." ABSOLUTE
3030
set(LMDBLIB_PKG "${REPO_ROOT}/native-packages/lmdblib")
3131
set(LMDBLIB_SRC "${LMDBLIB_PKG}/cpp/src")
3232
set(LMDBLIB_LIB "${LMDBLIB_PKG}/cpp/build/lib/liblmdblib.a")
33-
set(LMDB_INCLUDE "${LMDBLIB_PKG}/cpp/build/_deps/lmdb/src/lmdb_repo/libraries/liblmdb")
34-
set(LMDB_LIB "${LMDB_INCLUDE}/liblmdb.a")
33+
set(LMDB_INCLUDE "${LMDBLIB_PKG}/cpp/build/_deps/lmdb-src/libraries/liblmdb")
34+
set(LMDB_LIB "${LMDBLIB_PKG}/cpp/build/lib/liblmdb.a")
35+
# msgpack-c headers (header-only) come transitively from lmdblib's build: lmdblib fetches
36+
# msgpack-c and compiles liblmdblib.a against it, so the wire ABI is single-sourced rather
37+
# than re-fetched here. (Matches how the wsdb package consumes msgpack.)
38+
set(MSGPACK_INCLUDE "${LMDBLIB_PKG}/cpp/build/_deps/msgpack-src/include")
3539

36-
foreach(artifact "${LMDBLIB_LIB}" "${LMDB_LIB}")
40+
foreach(artifact "${LMDBLIB_LIB}" "${LMDB_LIB}" "${MSGPACK_INCLUDE}")
3741
if(NOT EXISTS "${artifact}")
3842
message(FATAL_ERROR "Missing prebuilt artifact: ${artifact}\n"
3943
"Build lmdblib first (cd native-packages/lmdblib && ./bootstrap.sh).")
4044
endif()
4145
endforeach()
4246

43-
# ---------------------------------------------------------------------------
44-
# msgpack-c headers (own copy; commit must match lmdblib's so the wire ABI lines up).
45-
# ---------------------------------------------------------------------------
46-
set(DEPS_PREFIX "${CMAKE_BINARY_DIR}/_deps")
47-
include(ExternalProject)
48-
49-
set(MSGPACK_PREFIX "${DEPS_PREFIX}/msgpack-c")
50-
set(MSGPACK_INCLUDE "${MSGPACK_PREFIX}/src/msgpack-c/include")
51-
ExternalProject_Add(
52-
msgpack-c
53-
PREFIX ${MSGPACK_PREFIX}
54-
DOWNLOAD_COMMAND
55-
sh -c "mkdir -p ${MSGPACK_PREFIX}/src/msgpack-c && cd ${MSGPACK_PREFIX}/src/msgpack-c && git init --quiet && (git remote add origin https://github.com/AztecProtocol/msgpack-c.git 2>/dev/null || true) && git fetch --depth 1 origin --quiet c0334576ed657fb3b3c49e8e61402989fb84146d && git reset --quiet --hard FETCH_HEAD"
56-
SOURCE_DIR ${MSGPACK_PREFIX}/src/msgpack-c
57-
CONFIGURE_COMMAND ""
58-
BUILD_COMMAND ""
59-
INSTALL_COMMAND ""
60-
UPDATE_COMMAND ""
61-
)
62-
6347
# ---------------------------------------------------------------------------
6448
# Node addon-api / node-api headers (resolved from this package's node_modules).
6549
# ---------------------------------------------------------------------------
@@ -101,7 +85,6 @@ set(KVDB_SOURCES
10185
src/util/promise.cpp
10286
)
10387
add_library(nodejs_module SHARED ${KVDB_SOURCES})
104-
add_dependencies(nodejs_module msgpack-c)
10588
set_target_properties(nodejs_module PROPERTIES PREFIX "" SUFFIX ".node")
10689
target_include_directories(nodejs_module PRIVATE ${NODE_API_HEADERS_DIR} ${NODE_ADDON_API_DIR})
10790

‎native-packages/kvdb/cpp/src/init_module.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
Napi::Object Init(Napi::Env env, Napi::Object exports) {
55
exports.Set(Napi::String::New(env, "LMDBStore"),
6-
bb::nodejs::lmdb_store::LMDBStoreWrapper::get_class(env));
6+
azteclabs::kvdb::lmdb_store::LMDBStoreWrapper::get_class(env));
77
return exports;
88
}
99

‎native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_message.hpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
#pragma once
22
#include "lmdblib/types.hpp"
33
#include "messaging/header.hpp"
4-
#include "serialization.hpp"
4+
#include <msgpack.hpp>
55
#include "msgpack/adaptor/define_decl.hpp"
66
#include <cstdint>
77
#include <optional>
88
#include <string>
99

10-
namespace bb::nodejs::lmdb_store {
10+
namespace azteclabs::kvdb::lmdb_store {
1111

12-
using namespace bb::messaging;
12+
using namespace azteclabs::kvdb::messaging;
1313

1414
enum LMDBStoreMessageType {
1515
OPEN_DATABASE = FIRST_APP_MSG_TYPE,
@@ -139,6 +139,6 @@ struct CopyStoreRequest {
139139
MSGPACK_DEFINE_MAP(dstPath, compact);
140140
};
141141

142-
} // namespace bb::nodejs::lmdb_store
142+
} // namespace azteclabs::kvdb::lmdb_store
143143

144-
MSGPACK_ADD_ENUM(bb::nodejs::lmdb_store::LMDBStoreMessageType)
144+
MSGPACK_ADD_ENUM(azteclabs::kvdb::lmdb_store::LMDBStoreMessageType)

‎native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_wrapper.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@
1111
#include <stdexcept>
1212
#include <utility>
1313

14-
using namespace bb::nodejs;
15-
using namespace bb::nodejs::lmdb_store;
14+
using namespace azteclabs::kvdb;
15+
using namespace azteclabs::kvdb::lmdb_store;
1616

1717
const uint64_t DEFAULT_MAP_SIZE = 1024UL * 1024;
1818
const uint64_t DEFAULT_MAX_READERS = 16;
@@ -113,7 +113,7 @@ void LMDBStoreWrapper::verify_store() const
113113
if (_store) {
114114
return;
115115
}
116-
throw std::runtime_error(bb::lmdblib::format("LMDB store unavailable, was close already called?"));
116+
throw std::runtime_error(azteclabs::lmdblib::format("LMDB store unavailable, was close already called?"));
117117
}
118118

119119
BoolResponse LMDBStoreWrapper::open_database(const OpenDatabaseRequest& req)
@@ -281,7 +281,7 @@ BoolResponse LMDBStoreWrapper::copy_store(const CopyStoreRequest& req)
281281
return { true };
282282
}
283283

284-
std::pair<bool, bb::lmdblib::KeyDupValuesVector> LMDBStoreWrapper::_advance_cursor(const lmdblib::LMDBCursor& cursor,
284+
std::pair<bool, azteclabs::lmdblib::KeyDupValuesVector> LMDBStoreWrapper::_advance_cursor(const lmdblib::LMDBCursor& cursor,
285285
bool reverse,
286286
uint64_t page_size)
287287
{

‎native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_wrapper.hpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
#include <mutex>
1313
#include <napi.h>
1414

15-
namespace bb::nodejs::lmdb_store {
15+
namespace azteclabs::kvdb::lmdb_store {
1616

1717
struct CursorData {
1818
lmdblib::LMDBCursor::SharedPtr cursor;
@@ -38,7 +38,7 @@ class LMDBStoreWrapper : public Napi::ObjectWrap<LMDBStoreWrapper> {
3838
std::mutex _cursor_mutex;
3939
std::unordered_map<uint64_t, CursorData> _cursors;
4040

41-
bb::nodejs::AsyncMessageProcessor _msg_processor;
41+
azteclabs::kvdb::AsyncMessageProcessor _msg_processor;
4242

4343
void verify_store() const;
4444

@@ -69,4 +69,4 @@ class LMDBStoreWrapper : public Napi::ObjectWrap<LMDBStoreWrapper> {
6969
const lmdblib::Key& end_key);
7070
};
7171

72-
} // namespace bb::nodejs::lmdb_store
72+
} // namespace azteclabs::kvdb::lmdb_store

‎native-packages/kvdb/cpp/src/messaging/dispatcher.hpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#pragma once
22

33
#include "messaging/header.hpp"
4-
#include "serialization.hpp"
4+
#include <msgpack.hpp>
55
#include <atomic>
66
#include <cstdint>
77
#include <functional>
@@ -12,7 +12,7 @@
1212
#include <utility>
1313
#include <vector>
1414

15-
namespace bb::messaging {
15+
namespace azteclabs::kvdb::messaging {
1616

1717
using message_handler =
1818
std::function<bool(msgpack::object &, msgpack::sbuffer &)>;
@@ -30,7 +30,7 @@ class MessageDispatcher {
3030
MessageDispatcher() = default;
3131

3232
bool on_new_data(msgpack::object &obj, msgpack::sbuffer &buffer) const {
33-
bb::messaging::HeaderOnlyMessage header;
33+
azteclabs::kvdb::messaging::HeaderOnlyMessage header;
3434
obj.convert(header);
3535

3636
auto iter = message_handlers.find(header.msgType);
@@ -56,4 +56,4 @@ class MessageDispatcher {
5656
}
5757
};
5858

59-
} // namespace bb::messaging
59+
} // namespace azteclabs::kvdb::messaging

‎native-packages/kvdb/cpp/src/messaging/header.hpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
#pragma once
2-
#include "serialization.hpp"
2+
#include <msgpack.hpp>
33
#include <cstdint>
44
#include <cstring>
55

6-
namespace bb::messaging {
6+
namespace azteclabs::kvdb::messaging {
77

88
enum SystemMsgTypes { TERMINATE = 0, PING = 1, PONG = 2 };
99

@@ -49,4 +49,4 @@ template <class T> struct TypedMessage {
4949
MSGPACK_DEFINE_MAP(msgType, header, value);
5050
};
5151

52-
} // namespace bb::messaging
52+
} // namespace azteclabs::kvdb::messaging

‎native-packages/kvdb/cpp/src/serialization.hpp‎

Lines changed: 0 additions & 14 deletions
This file was deleted.

0 commit comments

Comments
 (0)