refactor: decouple wsdb from barretenberg headers

wsdb compiles with zero barretenberg headers — its own FieldElement (32-byte
canonical) + a poseidon2 c_bind are the only barretenberg link. Forks the bb-free
common utils, moves merkle_tree Signal into wsdb, keeps lmdb keys
backward-compatible (little-endian uint256), switches lmdblib to upstream msgpack
+ FetchContent (bb-free), and wires the lmdblib/wsdb C++ tests into the CI test
engine. Squashed from the decouple branch (10 commits).

Author: charlielye

Makefile

@@ -55,7 +55,7 @@ endef
 
 # Fast bootstrap.
 fast: release-image barretenberg boxes playground docs aztec-up \
-		  bb-tests l1-contracts-tests yarn-project-tests boxes-tests playground-tests aztec-up-tests docs-tests noir-protocol-circuits-tests release-image-tests spartan claude-tests ipc-codegen-tests
+		  bb-tests l1-contracts-tests yarn-project-tests boxes-tests playground-tests aztec-up-tests docs-tests noir-protocol-circuits-tests release-image-tests spartan claude-tests ipc-codegen-tests lmdblib-tests wsdb-tests
 
 # Full bootstrap.
 full: fast bb-full-tests bb-cpp-full yarn-project-benches
@@ -326,7 +326,7 @@ ipc-runtime-cross: ipc-runtime ipc-runtime-cross-arm64-linux ipc-runtime-cross-a
 
 # lmdblib and kvdb are barretenberg-free: they build against their own deps
 # (lmdb, msgpack-c, node-addon-api) only, never bb.
-.PHONY: lmdblib kvdb
+.PHONY: lmdblib kvdb lmdblib-tests wsdb-tests
 lmdblib:
 	$(call build,$@,native-packages/lmdblib)
 
@@ -336,6 +336,15 @@ kvdb: lmdblib
 wsdb: ipc-codegen ipc-runtime bb-cpp-native lmdblib
 	$(call build,$@,native-packages/wsdb)
 
+# Native-package C++ tests (self-contained gtest binaries). kvdb has no C++ tests
+# (its NAPI is exercised by yarn-project's kv-store tests). wsdb_tests use no bb
+# headers; the bb-header parity/equivalence target is manual (WSDB_BUILD_BB_TESTS).
+lmdblib-tests: lmdblib
+	$(call test,$@,native-packages/lmdblib)
+
+wsdb-tests: wsdb
+	$(call test,$@,native-packages/wsdb)
+
 #==============================================================================
 # .claude tooling
 #==============================================================================

barretenberg/cpp/src/barretenberg/crypto/poseidon2/poseidon2_capi.cpp

@@ -0,0 +1,39 @@
+#include "barretenberg/crypto/poseidon2/poseidon2.hpp"
+#include "barretenberg/crypto/poseidon2/poseidon2_params.hpp"
+#include "barretenberg/ecc/curves/bn254/fr.hpp"
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
+// Flat C ABI over Poseidon2, for in-process consumers that must hash at high
+// frequency but want no dependency on barretenberg's C++ headers, field type,
+// stdlib, or build flags (e.g. the standalone world-state DB service). Field
+// elements cross as 32 canonical bytes (barretenberg's field serialization);
+// the montgomery form stays private to barretenberg.
+namespace {
+using FF = bb::fr;
+using Poseidon2Bn254 = bb::crypto::Poseidon2<bb::crypto::Poseidon2Bn254ScalarFieldParams>;
+} // namespace
+
+extern "C" {
+
+// Hash `n` field elements. `elems` is n*32 canonical bytes; `out` receives 32.
+void bb_poseidon2_hash(const uint8_t* elems, size_t n, uint8_t* out)
+{
+    std::vector<FF> inputs;
+    inputs.reserve(n);
+    for (size_t i = 0; i < n; ++i) {
+        inputs.push_back(FF::serialize_from_buffer(elems + (i * 32)));
+    }
+    FF::serialize_to_buffer(Poseidon2Bn254::hash(inputs), out);
+}
+
+// Hash { separator, lhs, rhs } — matches Poseidon2HashPolicy::hash_pair_with_separator,
+// the domain-separated internal-node hash used by the Aztec merkle trees.
+void bb_poseidon2_hash_pair_with_separator(uint64_t separator, const uint8_t* lhs, const uint8_t* rhs, uint8_t* out)
+{
+    std::vector<FF> inputs{ FF(separator), FF::serialize_from_buffer(lhs), FF::serialize_from_buffer(rhs) };
+    FF::serialize_to_buffer(Poseidon2Bn254::hash(inputs), out);
+}
+
+} // extern "C"

native-packages/kvdb/cpp/CMakeLists.txt

@@ -30,8 +30,8 @@ get_filename_component(REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../.." ABSOLUTE
 set(LMDBLIB_PKG "${REPO_ROOT}/native-packages/lmdblib")
 set(LMDBLIB_SRC "${LMDBLIB_PKG}/cpp/src")
 set(LMDBLIB_LIB "${LMDBLIB_PKG}/cpp/build/lib/liblmdblib.a")
-set(LMDB_INCLUDE "${LMDBLIB_PKG}/cpp/build/_deps/lmdb/src/lmdb_repo/libraries/liblmdb")
-set(LMDB_LIB "${LMDB_INCLUDE}/liblmdb.a")
+set(LMDB_INCLUDE "${LMDBLIB_PKG}/cpp/build/_deps/lmdb-src/libraries/liblmdb")
+set(LMDB_LIB "${LMDBLIB_PKG}/cpp/build/lib/liblmdb.a")
 
 foreach(artifact "${LMDBLIB_LIB}" "${LMDB_LIB}")
     if(NOT EXISTS "${artifact}")
@@ -52,7 +52,7 @@ ExternalProject_Add(
     msgpack-c
     PREFIX ${MSGPACK_PREFIX}
     DOWNLOAD_COMMAND
-        sh -c "mkdir -p ${MSGPACK_PREFIX}/src/msgpack-c && cd ${MSGPACK_PREFIX}/src/msgpack-c && git init --quiet && (git remote add origin https://github.com/AztecProtocol/msgpack-c.git 2>/dev/null || true) && git fetch --depth 1 origin --quiet c0334576ed657fb3b3c49e8e61402989fb84146d && git reset --quiet --hard FETCH_HEAD"
+        sh -c "mkdir -p ${MSGPACK_PREFIX}/src/msgpack-c && cd ${MSGPACK_PREFIX}/src/msgpack-c && git init --quiet && (git remote add origin https://github.com/msgpack/msgpack-c.git 2>/dev/null || true) && git fetch --depth 1 origin --quiet refs/tags/cpp-7.0.0 && git reset --quiet --hard FETCH_HEAD"
     SOURCE_DIR ${MSGPACK_PREFIX}/src/msgpack-c
     CONFIGURE_COMMAND ""
     BUILD_COMMAND ""

native-packages/kvdb/cpp/src/serialization.hpp

@@ -1,14 +1,4 @@
 #pragma once
 
-// msgpack-c entrypoint for kvdb. We use msgpack-c directly (no barretenberg
-// serialization framework). The AztecProtocol msgpack-c fork expects the
-// includer to provide THROW/RETHROW macros; define them to standard exception
-// handling.
-#ifndef THROW
-#define THROW throw
-#endif
-#ifndef RETHROW
-#define RETHROW throw
-#endif
-
+// msgpack-c entrypoint (header-only, exceptions enabled).
 #include <msgpack.hpp>

native-packages/lmdblib/bootstrap.sh

@@ -13,19 +13,31 @@ hash=$(hash_str \
 function build_native {
   local build_dir="cpp/build"
   CC=$(which clang) CXX=$(which clang++) cmake -S cpp -B "$build_dir" -G Ninja >/dev/null
-  cmake --build "$build_dir" --target lmdblib
+  cmake --build "$build_dir" --target lmdblib lmdblib_tests
 }
 
 function build {
   echo_header "lmdblib build"
   build_native
 }
 
+# Emit test commands for the CI test engine (one self-contained gtest binary).
+function test_cmds {
+  echo "$hash:CPUS=2:TIMEOUT=120s native-packages/lmdblib/cpp/build/lmdblib_tests"
+}
+
+# Manual: build then run the tests directly.
+function test {
+  echo_header "lmdblib test"
+  build
+  test_cmds | filter_test_cmds | parallelize
+}
+
 function clean {
   rm -rf cpp/build
 }
 
-export -f build_native build clean
+export -f build_native build test_cmds test clean
 
 case "$cmd" in
   "")

native-packages/lmdblib/cpp/CMakeLists.txt

@@ -1,17 +1,13 @@
-# Standalone build for lmdblib — the LMDB C++ wrapper.
+# lmdblib — a small C++ wrapper around LMDB.
 #
-# This package owns the lmdb wrapper that barretenberg no longer needs (the
-# in-memory merkle core is lmdb-free). It builds two static archives that the
-# kvdb NAPI addon and the wsdb service link against:
-#   - liblmdb.a    : the upstream LMDB C library (pinned to bb's commit)
-#   - liblmdblib.a : the bb::lmdblib C++ wrapper objects
+# Builds two static archives that the kvdb NAPI addon and the wsdb service link:
+#   - liblmdb.a    : the LMDB C library (built from source, pinned commit)
+#   - liblmdblib.a : the lmdblib C++ wrapper objects
 #
-# It is barretenberg-free: it includes no bb headers and links no bb archives. It
-# vendors its own msgpack-c + lmdb (commits pinned to barretenberg/cpp/cmake/*) and
-# defines the shared DBStats POD locally (src/lmdblib/types.hpp).
+# It is self-contained: it vendors LMDB and msgpack-c (header-only) via FetchContent.
 
 cmake_minimum_required(VERSION 3.24)
-# C is enabled so CMAKE_C_COMPILER is populated for lmdb's ExternalProject build.
+# C is enabled to compile the LMDB C sources.
 project(aztec_lmdblib CXX C)
 
 set(CMAKE_CXX_STANDARD 20)
@@ -21,66 +17,54 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
 
 # Default to Release (-O3 -DNDEBUG). lmdblib's templated/inline headers (serialise_key,
 # value_cmp, the msgpack adaptors) are compiled into both liblmdblib.a and into every
-# consumer that includes them (the kvdb .node, the wsdb service). Matching the consumers'
+# consumer that includes them (the kvdb addon, the wsdb service); matching the consumers'
 # Release build keeps that inline codegen identical and avoids ODR divergence.
 if(NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE Release)
 endif()
 
 # ---------------------------------------------------------------------------
-# Third-party deps (own copies; commits must match barretenberg/cpp/cmake/*).
+# Third-party dependencies (fetched at configure time).
 # ---------------------------------------------------------------------------
-set(DEPS_PREFIX "${CMAKE_BINARY_DIR}/_deps")
-include(ExternalProject)
+include(FetchContent)
 
-# lmdb: build our own liblmdb.a (pinned to bb's commit in cmake/lmdb.cmake).
-set(LMDB_PREFIX "${DEPS_PREFIX}/lmdb")
-set(LMDB_INCLUDE "${LMDB_PREFIX}/src/lmdb_repo/libraries/liblmdb")
-set(LMDB_LIB "${LMDB_INCLUDE}/liblmdb.a")
-ExternalProject_Add(
-    lmdb_repo
-    PREFIX ${LMDB_PREFIX}
-    DOWNLOAD_COMMAND
-        sh -c "mkdir -p ${LMDB_PREFIX}/src/lmdb_repo && cd ${LMDB_PREFIX}/src/lmdb_repo && git init --quiet && (git remote add origin https://github.com/LMDB/lmdb.git 2>/dev/null || true) && git fetch --depth 1 origin --quiet ddd0a773e2f44d38e4e31ec9ed81af81f4e4ccbb && git reset --quiet --hard FETCH_HEAD"
-    SOURCE_DIR ${LMDB_PREFIX}/src/lmdb_repo
-    BUILD_IN_SOURCE YES
-    CONFIGURE_COMMAND ""
-    BUILD_COMMAND ${CMAKE_COMMAND} -E env --unset=CFLAGS --unset=CXXFLAGS CC=${CMAKE_C_COMPILER}${CMAKE_C_COMPILER_ARG1} AR=${CMAKE_AR} make -e -C libraries/liblmdb XCFLAGS=-fPIC liblmdb.a
-    INSTALL_COMMAND ""
-    UPDATE_COMMAND ""
-    BUILD_BYPRODUCTS ${LMDB_LIB}
+# LMDB: a plain-Makefile C library (no CMake). Fetch the source and compile the two
+# translation units ourselves so they use this project's toolchain and flags.
+FetchContent_Declare(
+    lmdb
+    GIT_REPOSITORY https://github.com/LMDB/lmdb.git
+    GIT_TAG ddd0a773e2f44d38e4e31ec9ed81af81f4e4ccbb
 )
-add_library(lmdb STATIC IMPORTED GLOBAL)
-add_dependencies(lmdb lmdb_repo)
-set_target_properties(lmdb PROPERTIES IMPORTED_LOCATION ${LMDB_LIB})
+FetchContent_MakeAvailable(lmdb)
+set(LMDB_DIR ${lmdb_SOURCE_DIR}/libraries/liblmdb)
+add_library(lmdb STATIC ${LMDB_DIR}/mdb.c ${LMDB_DIR}/midl.c)
+target_include_directories(lmdb PUBLIC ${LMDB_DIR})
 
-# msgpack-c: header-only; fetch for includes (pinned to bb's commit in cmake/msgpack.cmake).
-set(MSGPACK_PREFIX "${DEPS_PREFIX}/msgpack-c")
-set(MSGPACK_INCLUDE "${MSGPACK_PREFIX}/src/msgpack-c/include")
-ExternalProject_Add(
-    msgpack-c
-    PREFIX ${MSGPACK_PREFIX}
-    DOWNLOAD_COMMAND
-        sh -c "mkdir -p ${MSGPACK_PREFIX}/src/msgpack-c && cd ${MSGPACK_PREFIX}/src/msgpack-c && git init --quiet && (git remote add origin https://github.com/AztecProtocol/msgpack-c.git 2>/dev/null || true) && git fetch --depth 1 origin --quiet c0334576ed657fb3b3c49e8e61402989fb84146d && git reset --quiet --hard FETCH_HEAD"
-    SOURCE_DIR ${MSGPACK_PREFIX}/src/msgpack-c
-    CONFIGURE_COMMAND ""
-    BUILD_COMMAND ""
-    INSTALL_COMMAND ""
-    UPDATE_COMMAND ""
+# msgpack-c: header-only. Fetch the source and use its include directory directly
+# (we don't build msgpack-c's own CMake targets/tests).
+FetchContent_Declare(
+    msgpack
+    GIT_REPOSITORY https://github.com/msgpack/msgpack-c.git
+    GIT_TAG cpp-7.0.0
 )
+FetchContent_GetProperties(msgpack)
+if(NOT msgpack_POPULATED)
+    FetchContent_Populate(msgpack)
+endif()
+set(MSGPACK_INCLUDE ${msgpack_SOURCE_DIR}/include)
 
-# lmdblib is barretenberg-free: it links no bb archives and includes no bb headers.
-# It is still compiled with libc++ and -march=skylake so its objects are ABI- and
-# arch-compatible when linked into consumers (wsdb, kvdb) alongside bb's archives.
-add_compile_definitions(MSGPACK_NO_BOOST MSGPACK_USE_STD_VARIANT_ADAPTOR NO_PAR_ALGOS)
+# msgpack-c is used header-only with exceptions (no boost). Compiled with libc++ and
+# -march=skylake so lmdblib's objects are ABI- and arch-compatible with the consumers
+# (kvdb, wsdb) they are linked into.
+add_compile_definitions(MSGPACK_NO_BOOST NO_PAR_ALGOS)
 add_compile_options(-march=skylake -fconstexpr-steps=100000000 -fbracket-depth=1024)
 add_compile_options(-Wno-deprecated-declarations -Wno-missing-field-initializers)
 add_compile_options(-stdlib=libc++)
 
 include_directories(
     ${CMAKE_CURRENT_SOURCE_DIR}/src
     ${MSGPACK_INCLUDE}
-    ${LMDB_INCLUDE}
+    ${LMDB_DIR}
 )
 
 # ---------------------------------------------------------------------------
@@ -100,4 +84,24 @@ set(LMDBLIB_SOURCES
     src/lmdblib/queries.cpp
 )
 add_library(lmdblib STATIC ${LMDBLIB_SOURCES})
-add_dependencies(lmdblib lmdb_repo msgpack-c)
+add_dependencies(lmdblib lmdb)
+
+# ---------------------------------------------------------------------------
+# Tests (vendored googletest; self-contained, no external dependencies).
+# ---------------------------------------------------------------------------
+option(LMDBLIB_BUILD_TESTS "Build lmdblib C++ tests" ON)
+if(LMDBLIB_BUILD_TESTS)
+    add_link_options(-stdlib=libc++)
+    FetchContent_Declare(
+        googletest
+        GIT_REPOSITORY https://github.com/google/googletest.git
+        GIT_TAG v1.13.0
+    )
+    set(INSTALL_GTEST OFF CACHE BOOL "" FORCE)
+    FetchContent_MakeAvailable(googletest)
+    add_executable(lmdblib_tests
+        src/lmdblib/lmdb_store.test.cpp
+        src/lmdblib/lmdb_environment.test.cpp
+    )
+    target_link_libraries(lmdblib_tests PRIVATE lmdblib lmdb GTest::gtest_main pthread)
+endif()

native-packages/lmdblib/cpp/src/lmdblib/lmdb_environment.test.cpp

@@ -11,9 +11,6 @@
 #include <thread>
 #include <vector>
 
-#include "barretenberg/common/serialize.hpp"
-#include "barretenberg/common/streams.hpp"
-#include "barretenberg/common/test.hpp"
 #include "lmdblib/fixtures.hpp"
 #include "lmdblib/lmdb_database.hpp"
 #include "lmdblib/lmdb_db_transaction.hpp"

native-packages/lmdblib/cpp/src/lmdblib/lmdb_store.test.cpp

@@ -12,9 +12,6 @@
 #include <thread>
 #include <vector>
 
-#include "barretenberg/common/serialize.hpp"
-#include "barretenberg/common/streams.hpp"
-#include "barretenberg/common/test.hpp"
 #include "lmdblib/fixtures.hpp"
 #include "lmdblib/lmdb_cursor.hpp"
 #include "lmdblib/lmdb_database.hpp"

native-packages/lmdblib/cpp/src/lmdblib/serialization.hpp

@@ -1,14 +1,4 @@
 #pragma once
 
-// msgpack-c entrypoint for lmdblib. We use msgpack-c directly (no barretenberg
-// serialization framework). The AztecProtocol msgpack-c fork expects the
-// includer to provide THROW/RETHROW macros; define them to standard exception
-// handling.
-#ifndef THROW
-#define THROW throw
-#endif
-#ifndef RETHROW
-#define RETHROW throw
-#endif
-
+// msgpack-c entrypoint for lmdblib (header-only, exceptions enabled).
 #include <msgpack.hpp>

native-packages/wsdb/bootstrap.sh

@@ -33,7 +33,7 @@ function generate_ts_package {
 function build_native {
   local build_dir="cpp/build"
   CC=$(which clang) CXX=$(which clang++) cmake -S cpp -B "$build_dir" -G Ninja >/dev/null
-  cmake --build "$build_dir" --target "$WSDB_BINARY"
+  cmake --build "$build_dir" --target "$WSDB_BINARY" wsdb_tests
   local target_dir="ts/build/$(arch)-$(os)"
   mkdir -p "$target_dir"
   cp "$build_dir/bin/$WSDB_BINARY" "$target_dir/$WSDB_BINARY"
@@ -48,6 +48,20 @@ function build {
   (cd ts && ./scripts/prepare_arch_packages.sh "$(arch)-$(os)=build/$(arch)-$(os)/$WSDB_BINARY")
 }
 
+# Emit test commands for the CI test engine. The decoupled wsdb_tests use no
+# barretenberg headers; they link libbarretenberg.a only for the poseidon2 c_bind.
+# (The optional bb-header parity/equivalence target, WSDB_BUILD_BB_TESTS, is manual.)
+function test_cmds {
+  echo "$hash:CPUS=8:TIMEOUT=600s native-packages/wsdb/cpp/build/bin/wsdb_tests"
+}
+
+# Manual: build then run the tests directly.
+function test {
+  echo_header "wsdb test"
+  build
+  test_cmds | filter_test_cmds | parallelize
+}
+
 function clean {
   rm -rf ts node_modules cpp/build
 }
@@ -64,7 +78,7 @@ function release {
   (cd ts && retry "deploy_npm ${REF_NAME#v}")
 }
 
-export -f generate_ts_package build_native build clean release
+export -f generate_ts_package build_native build test_cmds test clean release
 
 case "$cmd" in
   "")