Merge bitcoin-core/libmultiprocess#231 (Add windows support)

Sjors · Sjors · commit 8db8bf06ee55 · 2026-04-23T16:21:09.000+02:00
diff --git a/src/ipc/libmultiprocess/.github/workflows/bitcoin-core-ci.yml b/src/ipc/libmultiprocess/.github/workflows/bitcoin-core-ci.yml
@@ -18,6 +18,8 @@ concurrency:
 
 env:
   BITCOIN_REPO: bitcoin/bitcoin
+  # Temporary: use PR #35084 until it merges; revert to refs/heads/master after
+  BITCOIN_CORE_REF: refs/pull/35084/merge
   LLVM_VERSION: 22
   LIBCXX_DIR: /tmp/libcxx-build/
 
@@ -79,6 +81,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: ${{ env.BITCOIN_REPO }}
+          ref: ${{ env.BITCOIN_CORE_REF }}
           fetch-depth: 1
 
       - name: Checkout libmultiprocess
@@ -195,6 +198,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: ${{ env.BITCOIN_REPO }}
+          ref: ${{ env.BITCOIN_CORE_REF }}
           fetch-depth: 1
 
       - name: Checkout libmultiprocess
diff --git a/src/ipc/libmultiprocess/CMakeLists.txt b/src/ipc/libmultiprocess/CMakeLists.txt
@@ -13,7 +13,7 @@ endif()
 include("cmake/compat_find.cmake")
 
 find_package(Threads REQUIRED)
-find_package(CapnProto 0.7 QUIET NO_MODULE)
+find_package(CapnProto 0.9 QUIET NO_MODULE)
  if(NOT CapnProto_FOUND)
    message(FATAL_ERROR
      "Cap'n Proto is required but was not found.\n"
@@ -203,6 +203,10 @@ target_link_libraries(mpgen PRIVATE CapnProto::capnp-rpc)
 target_link_libraries(mpgen PRIVATE CapnProto::capnpc)
 target_link_libraries(mpgen PRIVATE CapnProto::kj)
 target_link_libraries(mpgen PRIVATE Threads::Threads)
+target_compile_definitions(mpgen PRIVATE
+  "CAPNP_EXECUTABLE=\"$<TARGET_FILE:CapnProto::capnp_tool>\""
+  "CAPNPC_CXX_EXECUTABLE=\"$<TARGET_FILE:CapnProto::capnpc_cpp>\""
+  "CAPNP_INCLUDE_DIRS=\"${CAPNP_INCLUDE_DIRS}\"")
 set_target_properties(mpgen PROPERTIES
     INSTALL_RPATH_USE_LINK_PATH TRUE)
 set_target_properties(mpgen PROPERTIES
diff --git a/src/ipc/libmultiprocess/ci/configs/olddeps.bash b/src/ipc/libmultiprocess/ci/configs/olddeps.bash
@@ -1,5 +1,5 @@
 CI_DESC="CI job using old Cap'n Proto and cmake versions"
 CI_DIR=build-olddeps
 export CXXFLAGS="-Werror -Wall -Wextra -Wpedantic -Wno-unused-parameter -Wno-error=array-bounds"
-NIX_ARGS=(--argstr capnprotoVersion "0.7.1" --argstr cmakeVersion "3.12.4")
+NIX_ARGS=(--argstr capnprotoVersion "0.9.2" --argstr cmakeVersion "3.12.4")
 BUILD_ARGS=(-k)
diff --git a/src/ipc/libmultiprocess/cmake/compat_config.cmake b/src/ipc/libmultiprocess/cmake/compat_config.cmake
@@ -12,6 +12,75 @@ if (NOT DEFINED capnp_PREFIX AND DEFINED CAPNP_INCLUDE_DIRS)
   get_filename_component(capnp_PREFIX "${CAPNP_INCLUDE_DIRS}" DIRECTORY)
 endif()
 
+if (NOT DEFINED CAPNP_INCLUDE_DIRS AND DEFINED capnp_PREFIX)
+  set(CAPNP_INCLUDE_DIRS "${capnp_PREFIX}/include")
+endif()
+
+if (NOT TARGET CapnProto::capnp_tool)
+  if (DEFINED CAPNP_EXECUTABLE)
+    add_executable(CapnProto::capnp_tool IMPORTED GLOBAL)
+    set_target_properties(CapnProto::capnp_tool PROPERTIES IMPORTED_LOCATION "${CAPNP_EXECUTABLE}")
+  elseif (DEFINED capnp_PREFIX)
+    add_executable(CapnProto::capnp_tool IMPORTED GLOBAL)
+    set_target_properties(CapnProto::capnp_tool PROPERTIES IMPORTED_LOCATION "${capnp_PREFIX}/bin/capnp")
+  endif()
+endif()
+
+if (NOT TARGET CapnProto::capnpc_cpp)
+  if (DEFINED CAPNPC_CXX_EXECUTABLE)
+    add_executable(CapnProto::capnpc_cpp IMPORTED GLOBAL)
+    set_target_properties(CapnProto::capnpc_cpp PROPERTIES IMPORTED_LOCATION "${CAPNPC_CXX_EXECUTABLE}")
+  elseif (DEFINED capnp_PREFIX)
+    add_executable(CapnProto::capnpc_cpp IMPORTED GLOBAL)
+    set_target_properties(CapnProto::capnpc_cpp PROPERTIES IMPORTED_LOCATION "${capnp_PREFIX}/bin/capnpc-c++")
+  endif()
+endif()
+
+# Validate CapnProto tool target locations and fix if broken.
+# Some packaged capnproto versions (e.g., Ubuntu Noble libcapnp-dev 1.0.1)
+# have incorrect IMPORTED_LOCATION paths due to a packaging bug where the cmake
+# config file is installed under /usr/lib/.../cmake/ but the _IMPORT_PREFIX
+# calculation goes up too few directory levels, yielding /usr/lib/bin/capnp
+# instead of the correct /usr/bin/capnp.
+foreach(_mp_tool IN ITEMS capnp_tool capnpc_cpp)
+  if (TARGET "CapnProto::${_mp_tool}")
+    get_target_property(_mp_configs "CapnProto::${_mp_tool}" IMPORTED_CONFIGURATIONS)
+    set(_mp_valid FALSE)
+    foreach(_mp_cfg IN LISTS _mp_configs)
+      get_target_property(_mp_loc "CapnProto::${_mp_tool}" "IMPORTED_LOCATION_${_mp_cfg}")
+      if (EXISTS "${_mp_loc}")
+        set(_mp_valid TRUE)
+        break()
+      endif()
+    endforeach()
+    if (NOT _mp_valid)
+      get_target_property(_mp_loc "CapnProto::${_mp_tool}" IMPORTED_LOCATION)
+      if (EXISTS "${_mp_loc}")
+        set(_mp_valid TRUE)
+      endif()
+    endif()
+    if (NOT _mp_valid)
+      if ("${_mp_tool}" STREQUAL "capnp_tool")
+        find_program(_mp_fixed capnp HINTS "${capnp_PREFIX}/bin")
+      else()
+        find_program(_mp_fixed capnpc-c++ HINTS "${capnp_PREFIX}/bin")
+      endif()
+      if (_mp_fixed)
+        foreach(_mp_cfg IN LISTS _mp_configs)
+          set_target_properties("CapnProto::${_mp_tool}" PROPERTIES "IMPORTED_LOCATION_${_mp_cfg}" "${_mp_fixed}")
+        endforeach()
+        set_target_properties("CapnProto::${_mp_tool}" PROPERTIES IMPORTED_LOCATION "${_mp_fixed}")
+      endif()
+      unset(_mp_fixed CACHE)
+    endif()
+  endif()
+endforeach()
+unset(_mp_tool)
+unset(_mp_configs)
+unset(_mp_valid)
+unset(_mp_cfg)
+unset(_mp_loc)
+
 if (NOT DEFINED CAPNPC_OUTPUT_DIR)
   set(CAPNPC_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 endif()
diff --git a/src/ipc/libmultiprocess/doc/design.md b/src/ipc/libmultiprocess/doc/design.md
@@ -120,7 +120,7 @@ sequenceDiagram
     participant PMT as ProxyMethodTraits
     participant Impl as Actual C++ Method
 
-    serverInvoke->>SF1: SF1::invoke 
+    serverInvoke->>SF1: SF1::invoke
     SF1->>SF2: SF2::invoke
     SF2->>SR: SR::invoke
     SR->>SC: SC::invoke
@@ -165,7 +165,7 @@ Thread mapping enables each client thread to have a dedicated server thread proc
 Thread mapping is initialized by defining an interface method with a `ThreadMap` parameter and/or response. The example below adds `ThreadMap` to the `construct` method because libmultiprocess calls the `construct` method automatically.
 
 ```capnp
-interface InitInterface $Proxy.wrap("Init") { 
+interface InitInterface $Proxy.wrap("Init") {
     construct @0 (threadMap: Proxy.ThreadMap) -> (threadMap :Proxy.ThreadMap);
 }
 ```
diff --git a/src/ipc/libmultiprocess/doc/versions.md b/src/ipc/libmultiprocess/doc/versions.md
@@ -7,8 +7,17 @@ Library versions are tracked with simple
 Versioning policy is described in the [version.h](../include/mp/version.h)
 include.
 
-## v10
+## v12
 - Current unstable version.
+- Adds support for nonunix platforms, making API changes that are not backwards compatible.
+
+## [v11.0](https://github.com/bitcoin-core/libmultiprocess/commits/v11.0)
+- Improves debug output if EventLoop::post callback fails.
+
+## [v10.0](https://github.com/bitcoin-core/libmultiprocess/commits/v10.0)
+- Increases spawn test timeout to avoid spurious failures.
+- Uses `throwRecoverableException` instead of raw `throw` to improve runtime error messages in macOS builds.
+- Used in Bitcoin Core master branch, pulled in by [#34977](https://github.com/bitcoin/bitcoin/pull/34977). Also pulled into Bitcoin Core 31.x stable branch by [#35028](https://github.com/bitcoin/bitcoin/pull/35028).
 
 ## [v9.0](https://github.com/bitcoin-core/libmultiprocess/commits/v9.0)
 - Fixes race conditions where worker thread could be used after destruction, where getParams() could be called after request cancel, and where m_on_cancel could be called after request finishes.
diff --git a/src/ipc/libmultiprocess/example/calculator.cpp b/src/ipc/libmultiprocess/example/calculator.cpp
@@ -6,8 +6,7 @@
 #include <init.capnp.h>
 #include <init.capnp.proxy.h> // NOLINT(misc-include-cleaner) // IWYU pragma: keep
 
-#include <charconv>
-#include <cstring>
+#include <cstring> // IWYU pragma: keep
 #include <fstream>
 #include <functional>
 #include <iostream>
@@ -16,9 +15,9 @@
 #include <kj/memory.h>
 #include <memory>
 #include <mp/proxy-io.h>
+#include <mp/util.h>
 #include <stdexcept>
 #include <string>
-#include <system_error>
 #include <utility>
 
 class CalculatorImpl : public Calculator
@@ -51,14 +50,10 @@ int main(int argc, char** argv)
         std::cout << "Usage: mpcalculator <fd>\n";
         return 1;
     }
-    int fd;
-    if (std::from_chars(argv[1], argv[1] + strlen(argv[1]), fd).ec != std::errc{}) {
-        std::cerr << argv[1] << " is not a number or is larger than an int\n";
-        return 1;
-    }
+    mp::SocketId socket{mp::StartSpawned(argv[1])};
     mp::EventLoop loop("mpcalculator", LogPrint);
     std::unique_ptr<Init> init = std::make_unique<InitImpl>();
-    mp::ServeStream<InitInterface>(loop, fd, *init);
+    mp::ServeStream<InitInterface>(loop, mp::MakeStream(loop.m_io_context, socket), *init);
     loop.loop();
     return 0;
 }
diff --git a/src/ipc/libmultiprocess/example/example.cpp b/src/ipc/libmultiprocess/example/example.cpp
@@ -19,20 +19,20 @@
 #include <string>
 #include <thread>
 #include <tuple>
+#include <utility>
 #include <vector>
 
 namespace fs = std::filesystem;
 
 static auto Spawn(mp::EventLoop& loop, const std::string& process_argv0, const std::string& new_exe_name)
 {
-    int pid;
-    const int fd = mp::SpawnProcess(pid, [&](int fd) -> std::vector<std::string> {
+    const auto [pid, socket] = mp::SpawnProcess([&](mp::ConnectInfo info) -> std::vector<std::string> {
         fs::path path = process_argv0;
         path.remove_filename();
         path.append(new_exe_name);
-        return {path.string(), std::to_string(fd)};
+        return {path.string(), std::move(info)};
     });
-    return std::make_tuple(mp::ConnectStream<InitInterface>(loop, fd), pid);
+    return std::make_tuple(mp::ConnectStream<InitInterface>(loop, mp::MakeStream(loop.m_io_context, socket)), pid);
 }
 
 static void LogPrint(mp::LogMessage log_data)
diff --git a/src/ipc/libmultiprocess/example/printer.cpp b/src/ipc/libmultiprocess/example/printer.cpp
@@ -7,18 +7,17 @@
 #include <init.capnp.h>
 #include <init.capnp.proxy.h> // NOLINT(misc-include-cleaner) // IWYU pragma: keep
 
-#include <charconv>
-#include <cstring>
+#include <cstring> // IWYU pragma: keep
 #include <fstream>
 #include <iostream>
 #include <kj/async.h>
 #include <kj/common.h>
 #include <kj/memory.h>
 #include <memory>
 #include <mp/proxy-io.h>
+#include <mp/util.h>
 #include <stdexcept>
 #include <string>
-#include <system_error>
 
 class PrinterImpl : public Printer
 {
@@ -44,14 +43,10 @@ int main(int argc, char** argv)
         std::cout << "Usage: mpprinter <fd>\n";
         return 1;
     }
-    int fd;
-    if (std::from_chars(argv[1], argv[1] + strlen(argv[1]), fd).ec != std::errc{}) {
-        std::cerr << argv[1] << " is not a number or is larger than an int\n";
-        return 1;
-    }
+    mp::SocketId socket{mp::StartSpawned(argv[1])};
     mp::EventLoop loop("mpprinter", LogPrint);
     std::unique_ptr<Init> init = std::make_unique<InitImpl>();
-    mp::ServeStream<InitInterface>(loop, fd, *init);
+    mp::ServeStream<InitInterface>(loop, mp::MakeStream(loop.m_io_context, socket), *init);
     loop.loop();
     return 0;
 }
diff --git a/src/ipc/libmultiprocess/include/mp/config.h.in b/src/ipc/libmultiprocess/include/mp/config.h.in
@@ -6,7 +6,6 @@
 #define MP_CONFIG_H
 
 #cmakedefine CMAKE_INSTALL_PREFIX "@CMAKE_INSTALL_PREFIX@"
-#cmakedefine capnp_PREFIX "@capnp_PREFIX@"
 #cmakedefine HAVE_KJ_FILESYSTEM
 
 #cmakedefine HAVE_PTHREAD_GETNAME_NP @HAVE_PTHREAD_GETNAME_NP@
diff --git a/src/ipc/libmultiprocess/include/mp/proxy-io.h b/src/ipc/libmultiprocess/include/mp/proxy-io.h
@@ -16,6 +16,7 @@
 #include <condition_variable>
 #include <functional>
 #include <kj/function.h>
+#include <kj/io.h>
 #include <map>
 #include <memory>
 #include <optional>
@@ -210,6 +211,21 @@ class Logger
 
 std::string LongThreadName(const char* exe_name);
 
+inline SocketId StreamSocketId(const Stream& stream)
+{
+    if (stream) KJ_IF_MAYBE(socket, stream->getFd()) return *socket;
+#ifdef WIN32
+    if (stream) KJ_IF_MAYBE(handle, stream->getWin32Handle()) return reinterpret_cast<SocketId>(*handle);
+#endif
+    throw std::logic_error("Stream socket unset");
+}
+
+//! Wrap a socket file descriptor as an async stream, taking ownership of the fd.
+inline Stream MakeStream(kj::AsyncIoContext& io_context, SocketId socket)
+{
+    return io_context.lowLevelProvider->wrapSocketFd(socket, kj::LowLevelAsyncIoProvider::TAKE_OWNERSHIP);
+}
+
 //! Event loop implementation.
 //!
 //! Cap'n Proto threading model is very simple: all I/O operations are
@@ -308,11 +324,12 @@ class EventLoop
     //! Callback functions to run on async thread.
     std::optional<CleanupList> m_async_fns MP_GUARDED_BY(m_mutex);
 
-    //! Pipe read handle used to wake up the event loop thread.
-    int m_wait_fd = -1;
+    //! Socket pair used to post and wait for wakeups to the event loop thread.
+    kj::Own<kj::AsyncIoStream> m_wait_stream;
+    kj::Own<kj::AsyncIoStream> m_post_stream;
 
-    //! Pipe write handle used to wake up the event loop thread.
-    int m_post_fd = -1;
+    //! Synchronous writer used to write to m_post_stream.
+    kj::Own<kj::OutputStream> m_post_writer;
 
     //! Number of clients holding references to ProxyServerBase objects that
     //! reference this event loop.
@@ -793,17 +810,15 @@ kj::Promise<T> ProxyServer<Thread>::post(Fn&& fn)
     return ret;
 }
 
-//! Given stream file descriptor, make a new ProxyClient object to send requests
-//! over the stream. Also create a new Connection object embedded in the
-//! client that is freed when the client is closed.
+//! Given a stream, make a new ProxyClient object to send requests over it.
+//! Also create a new Connection object embedded in the client that is freed
+//! when the client is closed.
 template <typename InitInterface>
-std::unique_ptr<ProxyClient<InitInterface>> ConnectStream(EventLoop& loop, int fd)
+std::unique_ptr<ProxyClient<InitInterface>> ConnectStream(EventLoop& loop, kj::Own<kj::AsyncIoStream> stream)
 {
     typename InitInterface::Client init_client(nullptr);
     std::unique_ptr<Connection> connection;
     loop.sync([&] {
-        auto stream =
-            loop.m_io_context.lowLevelProvider->wrapSocketFd(fd, kj::LowLevelAsyncIoProvider::TAKE_OWNERSHIP);
         connection = std::make_unique<Connection>(loop, kj::mv(stream));
         init_client = connection->m_rpc_system->bootstrap(ServerVatId().vat_id).castAs<InitInterface>();
         Connection* connection_ptr = connection.get();
@@ -851,13 +866,12 @@ void _Listen(EventLoop& loop, kj::Own<kj::ConnectionReceiver>&& listener, InitIm
         }));
 }
 
-//! Given stream file descriptor and an init object, handle requests on the
-//! stream by calling methods on the Init object.
+//! Given a stream and an init object, handle requests on the stream by calling
+//! methods on the Init object.
 template <typename InitInterface, typename InitImpl>
-void ServeStream(EventLoop& loop, int fd, InitImpl& init)
+void ServeStream(EventLoop& loop, kj::Own<kj::AsyncIoStream> stream, InitImpl& init)
 {
-    _Serve<InitInterface>(
-        loop, loop.m_io_context.lowLevelProvider->wrapSocketFd(fd, kj::LowLevelAsyncIoProvider::TAKE_OWNERSHIP), init);
+    _Serve<InitInterface>(loop, kj::mv(stream), init);
 }
 
 //! Given listening socket file descriptor and an init object, handle incoming
diff --git a/src/ipc/libmultiprocess/include/mp/proxy.h b/src/ipc/libmultiprocess/include/mp/proxy.h
@@ -314,11 +314,11 @@ static constexpr int FIELD_BOXED = 16;
 template <typename Field, int flags>
 struct Accessor : public Field
 {
-    static const bool in = flags & FIELD_IN;
-    static const bool out = flags & FIELD_OUT;
-    static const bool optional = flags & FIELD_OPTIONAL;
-    static const bool requested = flags & FIELD_REQUESTED;
-    static const bool boxed = flags & FIELD_BOXED;
+    static constexpr bool in = (flags & FIELD_IN) != 0;
+    static constexpr bool out = (flags & FIELD_OUT) != 0;
+    static constexpr bool optional = (flags & FIELD_OPTIONAL) != 0;
+    static constexpr bool requested = (flags & FIELD_REQUESTED) != 0;
+    static constexpr bool boxed = (flags & FIELD_BOXED) != 0;
 };
 
 //! Wrapper around std::function for passing std::function objects between client and servers.
diff --git a/src/ipc/libmultiprocess/include/mp/type-interface.h b/src/ipc/libmultiprocess/include/mp/type-interface.h
@@ -54,12 +54,12 @@ void CustomBuildField(TypeList<Impl&>,
     InvokeContext& invoke_context,
     Impl& value,
     Output&& output,
-    typename decltype(output.get())::Calls* enable = nullptr)
+    typename Decay<decltype(output.get())>::Calls* enable = nullptr)
 {
     // Disable deleter so proxy server object doesn't attempt to delete the
     // wrapped implementation when the proxy client is destroyed or
     // disconnected.
-    using Interface = typename decltype(output.get())::Calls;
+    using Interface = typename Decay<decltype(output.get())>::Calls;
     output.set(CustomMakeProxyServer<Interface, Impl>(invoke_context, std::shared_ptr<Impl>(&value, [](Impl*){})));
 }
 
diff --git a/src/ipc/libmultiprocess/include/mp/util.h b/src/ipc/libmultiprocess/include/mp/util.h
diff --git a/src/ipc/libmultiprocess/include/mp/version.h b/src/ipc/libmultiprocess/include/mp/version.h
diff --git a/src/ipc/libmultiprocess/src/mp/gen.cpp b/src/ipc/libmultiprocess/src/mp/gen.cpp
diff --git a/src/ipc/libmultiprocess/src/mp/proxy.cpp b/src/ipc/libmultiprocess/src/mp/proxy.cpp
diff --git a/src/ipc/libmultiprocess/src/mp/util.cpp b/src/ipc/libmultiprocess/src/mp/util.cpp
diff --git a/src/ipc/libmultiprocess/test/mp/test/spawn_tests.cpp b/src/ipc/libmultiprocess/test/mp/test/spawn_tests.cpp
