refactor: decouple wsdb from barretenberg headers

wsdb compiles with zero barretenberg headers — its own FieldElement (32-byte
canonical) + a poseidon2 c_bind are the only barretenberg link. Forks the bb-free
common utils, moves merkle_tree Signal into wsdb, keeps lmdb keys
backward-compatible (little-endian uint256), switches lmdblib to upstream msgpack
+ FetchContent (bb-free), and wires the lmdblib/wsdb C++ tests into the CI test
engine. Squashed from the decouple branch (10 commits).

Author: charlielye

Makefile

@@ -55,7 +55,7 @@ endef
 
 # Fast bootstrap.
 fast: release-image barretenberg boxes playground docs aztec-up \
-		  bb-tests l1-contracts-tests yarn-project-tests boxes-tests playground-tests aztec-up-tests docs-tests noir-protocol-circuits-tests release-image-tests spartan claude-tests ipc-codegen-tests
+		  bb-tests l1-contracts-tests yarn-project-tests boxes-tests playground-tests aztec-up-tests docs-tests noir-protocol-circuits-tests release-image-tests spartan claude-tests ipc-codegen-tests lmdblib-tests wsdb-tests
 
 # Full bootstrap.
 full: fast bb-full-tests bb-cpp-full yarn-project-benches
@@ -326,7 +326,7 @@ ipc-runtime-cross: ipc-runtime ipc-runtime-cross-arm64-linux ipc-runtime-cross-a
 
 # lmdblib and kvdb are barretenberg-free: they build against their own deps
 # (lmdb, msgpack-c, node-addon-api) only, never bb.
-.PHONY: lmdblib kvdb
+.PHONY: lmdblib kvdb lmdblib-tests wsdb-tests
 lmdblib:
 	$(call build,$@,native-packages/lmdblib)
 
@@ -336,6 +336,15 @@ kvdb: lmdblib
 wsdb: ipc-codegen ipc-runtime bb-cpp-native lmdblib
 	$(call build,$@,native-packages/wsdb)
 
+# Native-package C++ tests (self-contained gtest binaries). kvdb has no C++ tests
+# (its NAPI is exercised by yarn-project's kv-store tests). wsdb_tests use no bb
+# headers; the bb-header parity/equivalence target is manual (WSDB_BUILD_BB_TESTS).
+lmdblib-tests: lmdblib
+	$(call test,$@,native-packages/lmdblib)
+
+wsdb-tests: wsdb
+	$(call test,$@,native-packages/wsdb)
+
 #==============================================================================
 # .claude tooling
 #==============================================================================

barretenberg/cpp/src/barretenberg/crypto/poseidon2/poseidon2_capi.cpp

@@ -0,0 +1,39 @@
+#include "barretenberg/crypto/poseidon2/poseidon2.hpp"
+#include "barretenberg/crypto/poseidon2/poseidon2_params.hpp"
+#include "barretenberg/ecc/curves/bn254/fr.hpp"
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
+// Flat C ABI over Poseidon2, for in-process consumers that must hash at high
+// frequency but want no dependency on barretenberg's C++ headers, field type,
+// stdlib, or build flags (e.g. the standalone world-state DB service). Field
+// elements cross as 32 canonical bytes (barretenberg's field serialization);
+// the montgomery form stays private to barretenberg.
+namespace {
+using FF = bb::fr;
+using Poseidon2Bn254 = bb::crypto::Poseidon2<bb::crypto::Poseidon2Bn254ScalarFieldParams>;
+} // namespace
+
+extern "C" {
+
+// Hash `n` field elements. `elems` is n*32 canonical bytes; `out` receives 32.
+void bb_poseidon2_hash(const uint8_t* elems, size_t n, uint8_t* out)
+{
+    std::vector<FF> inputs;
+    inputs.reserve(n);
+    for (size_t i = 0; i < n; ++i) {
+        inputs.push_back(FF::serialize_from_buffer(elems + (i * 32)));
+    }
+    FF::serialize_to_buffer(Poseidon2Bn254::hash(inputs), out);
+}
+
+// Hash { separator, lhs, rhs } — matches Poseidon2HashPolicy::hash_pair_with_separator,
+// the domain-separated internal-node hash used by the Aztec merkle trees.
+void bb_poseidon2_hash_pair_with_separator(uint64_t separator, const uint8_t* lhs, const uint8_t* rhs, uint8_t* out)
+{
+    std::vector<FF> inputs{ FF(separator), FF::serialize_from_buffer(lhs), FF::serialize_from_buffer(rhs) };
+    FF::serialize_to_buffer(Poseidon2Bn254::hash(inputs), out);
+}
+
+} // extern "C"

barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.cpp

@@ -9,7 +9,6 @@ namespace bb::avm2::simulation {
 
 // Wire <-> domain conversion helpers are shared with the server handlers
 // (see wsdb_handlers.cpp) so both sides use the same encoding boundary.
-using bb::wsdb::Fr;
 using bb::wsdb::fr_from_wire;
 using bb::wsdb::fr_to_wire;
 using bb::wsdb::fr_vec_from_wire;
@@ -21,6 +20,7 @@ using bb::wsdb::revision_to_wire;
 using bb::wsdb::sequential_nullifier_from_wire;
 using bb::wsdb::sequential_public_data_from_wire;
 using bb::wsdb::tree_id_to_wire;
+using bb::wsdb::wire::Fr;
 
 // ---------------------------------------------------------------------------
 // Constructor

barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_wire_convert_client.hpp

@@ -19,77 +19,77 @@
 
 namespace bb::wsdb {
 
-inline Fr fr_to_wire(const bb::fr& d)
+inline wire::Fr fr_to_wire(const bb::fr& d)
 {
-    Fr r{};
+    wire::Fr r{};
     bb::fr::serialize_to_buffer(d, r.data());
     return r;
 }
 
-inline bb::fr fr_from_wire(const Fr& w)
+inline bb::fr fr_from_wire(const wire::Fr& w)
 {
     return bb::fr::serialize_from_buffer(w.data());
 }
 
-inline BlockHeaderHash block_header_hash_to_wire(const bb::fr& d)
+inline wire::BlockHeaderHash block_header_hash_to_wire(const bb::fr& d)
 {
-    BlockHeaderHash r{};
+    wire::BlockHeaderHash r{};
     bb::fr::serialize_to_buffer(d, r.data());
     return r;
 }
 
-inline bb::fr block_header_hash_from_wire(const BlockHeaderHash& w)
+inline bb::fr block_header_hash_from_wire(const wire::BlockHeaderHash& w)
 {
     return bb::fr::serialize_from_buffer(w.data());
 }
 
-inline PublicDataSlot public_data_slot_to_wire(const bb::fr& d)
+inline wire::PublicDataSlot public_data_slot_to_wire(const bb::fr& d)
 {
-    PublicDataSlot r{};
+    wire::PublicDataSlot r{};
     bb::fr::serialize_to_buffer(d, r.data());
     return r;
 }
 
-inline bb::fr public_data_slot_from_wire(const PublicDataSlot& w)
+inline bb::fr public_data_slot_from_wire(const wire::PublicDataSlot& w)
 {
     return bb::fr::serialize_from_buffer(w.data());
 }
 
-inline PublicDataValue public_data_value_to_wire(const bb::fr& d)
+inline wire::PublicDataValue public_data_value_to_wire(const bb::fr& d)
 {
-    PublicDataValue r{};
+    wire::PublicDataValue r{};
     bb::fr::serialize_to_buffer(d, r.data());
     return r;
 }
 
-inline bb::fr public_data_value_from_wire(const PublicDataValue& w)
+inline bb::fr public_data_value_from_wire(const wire::PublicDataValue& w)
 {
     return bb::fr::serialize_from_buffer(w.data());
 }
 
-inline Nullifier nullifier_to_wire(const bb::fr& d)
+inline wire::Nullifier nullifier_to_wire(const bb::fr& d)
 {
-    Nullifier r{};
+    wire::Nullifier r{};
     bb::fr::serialize_to_buffer(d, r.data());
     return r;
 }
 
-inline bb::fr nullifier_from_wire(const Nullifier& w)
+inline bb::fr nullifier_from_wire(const wire::Nullifier& w)
 {
     return bb::fr::serialize_from_buffer(w.data());
 }
 
-inline std::vector<Fr> fr_vec_to_wire(const std::vector<bb::fr>& d)
+inline std::vector<wire::Fr> fr_vec_to_wire(const std::vector<bb::fr>& d)
 {
-    std::vector<Fr> r;
+    std::vector<wire::Fr> r;
     r.reserve(d.size());
     for (const auto& x : d) {
         r.push_back(fr_to_wire(x));
     }
     return r;
 }
 
-inline std::vector<bb::fr> fr_vec_from_wire(const std::vector<Fr>& w)
+inline std::vector<bb::fr> fr_vec_from_wire(const std::vector<wire::Fr>& w)
 {
     std::vector<bb::fr> r;
     r.reserve(w.size());
@@ -117,12 +117,12 @@ inline crypto::merkle_tree::WorldStateRevision revision_from_wire(const wire::Wo
     };
 }
 
-inline MerkleTreeId tree_id_to_wire(crypto::merkle_tree::MerkleTreeId d)
+inline wire::MerkleTreeId tree_id_to_wire(crypto::merkle_tree::MerkleTreeId d)
 {
-    return static_cast<MerkleTreeId>(d);
+    return static_cast<wire::MerkleTreeId>(d);
 }
 
-inline crypto::merkle_tree::MerkleTreeId tree_id_from_wire(MerkleTreeId w)
+inline crypto::merkle_tree::MerkleTreeId tree_id_from_wire(wire::MerkleTreeId w)
 {
     return static_cast<crypto::merkle_tree::MerkleTreeId>(w);
 }

ipc-codegen/echo_example/cpp/src/echo_client.cpp

@@ -17,8 +17,8 @@
   } while (0)
 
 namespace {
-echo::Fr test_hash(uint8_t base) {
-  echo::Fr hash{};
+echo::wire::Fr test_hash(uint8_t base) {
+  echo::wire::Fr hash{};
   for (size_t i = 0; i < hash.size(); ++i) {
     hash[i] = static_cast<uint8_t>(base + i);
   }
@@ -74,7 +74,7 @@ int main(int argc, char **argv) {
     CHECK(resp.treeId == 7, "EchoAliases treeId");
     CHECK(resp.hash == hash, "EchoAliases hash");
     CHECK(resp.maybeHash == second, "EchoAliases maybeHash");
-    CHECK((resp.hashes == std::vector<echo::Fr>{hash, second}),
+    CHECK((resp.hashes == std::vector<echo::wire::Fr>{hash, second}),
           "EchoAliases hashes");
     std::cerr << "echo_client(cpp): EchoAliases OK\n";
   }

ipc-codegen/echo_example/cpp/src/golden_test.cpp

@@ -151,12 +151,12 @@ void check_response(const std::string &dir, const std::string &file,
   }
 }
 
-echo::Fr test_hash(uint8_t base) {
+echo::wire::Fr test_hash(uint8_t base) {
   std::array<uint8_t, 32> bytes{};
   for (size_t i = 0; i < bytes.size(); i++) {
     bytes[i] = static_cast<uint8_t>(base + i);
   }
-  return echo::Fr(bytes);
+  return echo::wire::Fr(bytes);
 }
 
 } // namespace

ipc-codegen/src/cpp_codegen.ts

@@ -550,10 +550,10 @@ template <typename Tag> struct Bin32Alias {
 
 namespace ${ns} {
 
-${aliasDecls}
-
 ${this.opts.wireNamespace ? `namespace ${this.opts.wireNamespace} {` : ""}
 
+${aliasDecls}
+
 ${structs}
 
 ${this.opts.wireNamespace ? `} // namespace ${this.opts.wireNamespace}` : ""}

native-packages/kvdb/cpp/CMakeLists.txt

@@ -30,36 +30,20 @@ get_filename_component(REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../.." ABSOLUTE
 set(LMDBLIB_PKG "${REPO_ROOT}/native-packages/lmdblib")
 set(LMDBLIB_SRC "${LMDBLIB_PKG}/cpp/src")
 set(LMDBLIB_LIB "${LMDBLIB_PKG}/cpp/build/lib/liblmdblib.a")
-set(LMDB_INCLUDE "${LMDBLIB_PKG}/cpp/build/_deps/lmdb/src/lmdb_repo/libraries/liblmdb")
-set(LMDB_LIB "${LMDB_INCLUDE}/liblmdb.a")
+set(LMDB_INCLUDE "${LMDBLIB_PKG}/cpp/build/_deps/lmdb-src/libraries/liblmdb")
+set(LMDB_LIB "${LMDBLIB_PKG}/cpp/build/lib/liblmdb.a")
+# msgpack-c headers (header-only) come transitively from lmdblib's build: lmdblib fetches
+# msgpack-c and compiles liblmdblib.a against it, so the wire ABI is single-sourced rather
+# than re-fetched here. (Matches how the wsdb package consumes msgpack.)
+set(MSGPACK_INCLUDE "${LMDBLIB_PKG}/cpp/build/_deps/msgpack-src/include")
 
-foreach(artifact "${LMDBLIB_LIB}" "${LMDB_LIB}")
+foreach(artifact "${LMDBLIB_LIB}" "${LMDB_LIB}" "${MSGPACK_INCLUDE}")
     if(NOT EXISTS "${artifact}")
         message(FATAL_ERROR "Missing prebuilt artifact: ${artifact}\n"
                             "Build lmdblib first (cd native-packages/lmdblib && ./bootstrap.sh).")
     endif()
 endforeach()
 
-# ---------------------------------------------------------------------------
-# msgpack-c headers (own copy; commit must match lmdblib's so the wire ABI lines up).
-# ---------------------------------------------------------------------------
-set(DEPS_PREFIX "${CMAKE_BINARY_DIR}/_deps")
-include(ExternalProject)
-
-set(MSGPACK_PREFIX "${DEPS_PREFIX}/msgpack-c")
-set(MSGPACK_INCLUDE "${MSGPACK_PREFIX}/src/msgpack-c/include")
-ExternalProject_Add(
-    msgpack-c
-    PREFIX ${MSGPACK_PREFIX}
-    DOWNLOAD_COMMAND
-        sh -c "mkdir -p ${MSGPACK_PREFIX}/src/msgpack-c && cd ${MSGPACK_PREFIX}/src/msgpack-c && git init --quiet && (git remote add origin https://github.com/AztecProtocol/msgpack-c.git 2>/dev/null || true) && git fetch --depth 1 origin --quiet c0334576ed657fb3b3c49e8e61402989fb84146d && git reset --quiet --hard FETCH_HEAD"
-    SOURCE_DIR ${MSGPACK_PREFIX}/src/msgpack-c
-    CONFIGURE_COMMAND ""
-    BUILD_COMMAND ""
-    INSTALL_COMMAND ""
-    UPDATE_COMMAND ""
-)
-
 # ---------------------------------------------------------------------------
 # Node addon-api / node-api headers (resolved from this package's node_modules).
 # ---------------------------------------------------------------------------
@@ -101,7 +85,6 @@ set(KVDB_SOURCES
     src/util/promise.cpp
 )
 add_library(nodejs_module SHARED ${KVDB_SOURCES})
-add_dependencies(nodejs_module msgpack-c)
 set_target_properties(nodejs_module PROPERTIES PREFIX "" SUFFIX ".node")
 target_include_directories(nodejs_module PRIVATE ${NODE_API_HEADERS_DIR} ${NODE_ADDON_API_DIR})
 

native-packages/kvdb/cpp/src/init_module.cpp

@@ -3,7 +3,7 @@
 
 Napi::Object Init(Napi::Env env, Napi::Object exports) {
   exports.Set(Napi::String::New(env, "LMDBStore"),
-              bb::nodejs::lmdb_store::LMDBStoreWrapper::get_class(env));
+              azteclabs::kvdb::lmdb_store::LMDBStoreWrapper::get_class(env));
   return exports;
 }
 

native-packages/kvdb/cpp/src/lmdb_store/lmdb_store_message.hpp

@@ -1,15 +1,15 @@
 #pragma once
 #include "lmdblib/types.hpp"
 #include "messaging/header.hpp"
-#include "serialization.hpp"
+#include <msgpack.hpp>
 #include "msgpack/adaptor/define_decl.hpp"
 #include <cstdint>
 #include <optional>
 #include <string>
 
-namespace bb::nodejs::lmdb_store {
+namespace azteclabs::kvdb::lmdb_store {
 
-using namespace bb::messaging;
+using namespace azteclabs::kvdb::messaging;
 
 enum LMDBStoreMessageType {
     OPEN_DATABASE = FIRST_APP_MSG_TYPE,
@@ -139,6 +139,6 @@ struct CopyStoreRequest {
     MSGPACK_DEFINE_MAP(dstPath, compact);
 };
 
-} // namespace bb::nodejs::lmdb_store
+} // namespace azteclabs::kvdb::lmdb_store
 
-MSGPACK_ADD_ENUM(bb::nodejs::lmdb_store::LMDBStoreMessageType)
+MSGPACK_ADD_ENUM(azteclabs::kvdb::lmdb_store::LMDBStoreMessageType)