db: update libmdbx v0.39.6

.circleci/config.yml

@@ -95,7 +95,7 @@ commands:
           command: |
             if [[ "<<parameters.compiler_id>>" == "clang" ]]
             then
-                conan_profile=linux_x64_clang_16_release
+                conan_profile=linux_x64_clang_18_release
             else
                 conan_profile=linux_x64_gcc_11_release
             fi
@@ -132,7 +132,7 @@ commands:
       - run:
           name: "CMake Fuzzer"
           working_directory: ~/build
-          command: cmake ../project -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_16_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON
+          command: cmake ../project -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_18_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON
       - run:
           name: "Build Fuzzer"
           command: |
@@ -256,7 +256,7 @@ jobs:
       - build_using_conan:
           build_type: Debug
           compiler_id: clang
-          compiler_version: 17
+          compiler_version: 18
       - test:
           sanitizer: address,undefined
 
@@ -415,7 +415,7 @@ jobs:
 parameters:
   clang_version_min:
     type: integer
-    default: 16
+    default: 18
   clang_version_latest:
     type: integer
     default: 18

.github/actions/fuzzer-common-steps/action.yml

@@ -11,7 +11,7 @@ runs:
   steps:
     - name: Install Compiler
       shell: bash
-      run: cmake/setup/compiler_install.sh clang 16
+      run: cmake/setup/compiler_install.sh clang 18
 
     - name: Clean Build Directory
       shell: bash
@@ -28,7 +28,7 @@ runs:
     - name: Configure CMake
       shell: bash
       working-directory: ${{runner.workspace}}/silkworm/build
-      run: CC=clang-16 CXX=clang++-16 cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_16_release -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON -DSILKWORM_FUZZER_SANITIZERS=${{inputs.fuzzer_sanitizers}}
+      run: CC=clang-16 CXX=clang++-16 cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_18_release -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON -DSILKWORM_FUZZER_SANITIZERS=${{inputs.fuzzer_sanitizers}}
 
     - name: Build Silkworm Fuzzer Test
       shell: bash

cmake/conan_quiet.cmake

@@ -8,7 +8,6 @@ set(GTest_FIND_QUIETLY YES)
 set(LibtorrentRasterbar_FIND_QUIETLY YES)
 set(Snappy_FIND_QUIETLY YES)
 set(Microsoft.GSL_FIND_QUIETLY YES)
-set(OpenSSL_FIND_QUIETLY YES)
 set(fmt_FIND_QUIETLY YES)
 set(roaring_FIND_QUIETLY YES)
 set(BZip2_FIND_QUIETLY YES)
@@ -25,15 +24,11 @@ set(asio-grpc_FIND_QUIETLY YES)
 set(benchmark_FIND_QUIETLY YES)
 set(gmp_FIND_QUIETLY YES)
 set(Boost_FIND_QUIETLY YES)
-set(SQLite3_FIND_QUIETLY YES)
 set(mimalloc_FIND_QUIETLY YES)
 set(re2_FIND_QUIETLY YES)
 set(gRPC_FIND_QUIETLY YES)
 set(protobuf_FIND_QUIETLY YES)
-set(ZLIB_FIND_QUIETLY YES)
 set(Protobuf_FIND_QUIETLY YES)
-set(BZip2_FIND_QUIETLY YES)
 set(SQLiteCpp_FIND_QUIETLY YES)
-set(GTest_FIND_QUIETLY YES)
 set(nlohmann_json_FIND_QUIETLY YES)
 set(libdeflate_FIND_QUIETLY YES)

cmake/profiles/linux_x64_clang_16_debug → cmake/profiles/linux_x64_clang_18_debug

@@ -2,7 +2,7 @@
 os=Linux
 arch=x86_64
 compiler=clang
-compiler.version=16
+compiler.version=18
 compiler.libcxx=libc++
 compiler.cppstd=17
 build_type=Debug

cmake/profiles/linux_x64_clang_16_release → cmake/profiles/linux_x64_clang_18_release

@@ -2,7 +2,7 @@
 os=Linux
 arch=x86_64
 compiler=clang
-compiler.version=16
+compiler.version=18
 compiler.libcxx=libc++
 compiler.cppstd=17
 build_type=Release

docs/fuzzer.md

@@ -6,7 +6,7 @@ To build the fuzzer use the following:
 ```bash
 mkdir build
 cd build
-cmake ../project -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_16_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON
+cmake ../project -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_18_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON
 cmake --build --target rpcdaemon_fuzzer_test
 cmake –-build --target rpcdaemon_fuzzer_diagnostics
 ```

silkworm/db/datastore/kvdb/mdbx.cpp

@@ -21,7 +21,15 @@ namespace detail {
     }
 
     std::string slice_as_hex(const Slice& data) {
-        return ::mdbx::to_hex(data).as_string();
+        return std::string(::mdbx::to_hex(data).as_string());
+    }
+
+    std::string slice_as_string(const Slice& data) {
+        return std::string(data.as_string());
+    }
+
+    silkworm::Bytes slice_as_bytes(const Slice& data) {
+        return silkworm::Bytes{reinterpret_cast<const unsigned char*>(data.data()), data.size()};
     }
 
     log::Args log_args_for_commit_latency(const MDBX_commit_latency& commit_latency) {
@@ -106,7 +114,7 @@ ::mdbx::env_managed open_env(const EnvConfig& config) {
         throw std::runtime_error("Database map size is too small. Min required " + human_size(db_file_size));
     }
 
-    uint32_t flags{MDBX_NOTLS | MDBX_NORDAHEAD | MDBX_COALESCE | MDBX_SYNC_DURABLE};  // Default flags
+    uint32_t flags{MDBX_NORDAHEAD | MDBX_SYNC_DURABLE};  // Default flags
 
     if (config.read_ahead) {
         flags &= ~MDBX_NORDAHEAD;
@@ -133,6 +141,9 @@ ::mdbx::env_managed open_env(const EnvConfig& config) {
     if (config.shared) {
         flags |= MDBX_ACCEDE;
     }
+    if (config.no_sticky_threads) {
+        flags |= MDBX_NOSTICKYTHREADS;
+    }
     if (config.write_map) {
         flags |= MDBX_WRITEMAP;
     }
@@ -574,7 +585,7 @@ std::vector<std::string> list_maps(::mdbx::txn& tx, bool throw_notfound) {
     ::mdbx::map_handle main_map{1};
     auto main_cursor{tx.open_cursor(main_map)};
     for (auto it{main_cursor.to_first(throw_notfound)}; it.done; it = main_cursor.to_next(throw_notfound)) {
-        map_names.push_back(it.key.as_string());
+        map_names.push_back(detail::slice_as_string(it.key));
     }
     return map_names;
 }

silkworm/db/datastore/kvdb/mdbx.hpp

@@ -60,6 +60,8 @@ namespace detail {
 
     std::string dump_mdbx_result(const CursorResult& result);
     std::string slice_as_hex(const Slice& data);
+    std::string slice_as_string(const Slice& data);
+    silkworm::Bytes slice_as_bytes(const Slice& data);
 }  // namespace detail
 
 class ROTxn;
@@ -367,6 +369,7 @@ struct EnvConfig {
     bool in_memory{false};              // Whether this db is in memory
     bool shared{false};                 // Whether this process opens a db already opened by another process
     bool read_ahead{false};             // Whether to enable mdbx read ahead
+    bool no_sticky_threads{false};      // Whether to allow moving db transactions between threads
     bool write_map{false};              // Whether to enable mdbx write map
     size_t page_size{os::page_size()};  // Mdbx page size
     size_t max_size{3_Tebi};            // Mdbx max map size

silkworm/db/datastore/kvdb/memory_mutation_cursor.cpp

@@ -523,6 +523,14 @@ CursorResult MemoryMutationCursor::to_current_next_multi() {
 }
 
 CursorResult MemoryMutationCursor::to_current_next_multi(bool throw_notfound) {
+    if (config_.value_mode == ::mdbx::value_mode::single) {
+        if (throw_notfound) {
+            throw_error_notfound();
+        } else {
+            return CursorResult{{}, {}, false};
+        }
+    }
+
     if (is_table_cleared()) {
         return memory_cursor_->to_current_next_multi(throw_notfound);
     }
@@ -598,7 +606,7 @@ CursorResult MemoryMutationCursor::find_multivalue(const Slice& key, const Slice
     const auto memory_result = memory_cursor_->find_multivalue(key, value, false);
 
     auto db_result = cursor_->find_multivalue(key, value, false);
-    if (db_result.key && is_entry_deleted(db_result.key, db_result.value)) {
+    if (db_result.done && is_entry_deleted(db_result.key, db_result.value)) {
         db_result = next_on_db(MoveType::kNextDup, throw_notfound);
     }
 
@@ -702,7 +710,7 @@ CursorResult MemoryMutationCursor::next_on_db(MemoryMutationCursor::MoveType typ
     CursorResult result = next_by_type(type, throw_notfound);
     if (!result.done) return result;
 
-    while (result.key && result.value && is_entry_deleted(result.key, result.value)) {
+    while (result.done && result.value && is_entry_deleted(result.key, result.value)) {
         result = next_by_type(type, throw_notfound);
         if (!result.done) return result;
     }
@@ -711,19 +719,27 @@ CursorResult MemoryMutationCursor::next_on_db(MemoryMutationCursor::MoveType typ
 }
 
 CursorResult MemoryMutationCursor::next_by_type(MemoryMutationCursor::MoveType type, bool throw_notfound) {
-    switch (type) {
-        case MoveType::kNext: {
-            return cursor_->to_next(throw_notfound);
-        }
-        case MoveType::kNextDup: {
-            return cursor_->to_current_next_multi(throw_notfound);
-        }
-        case MoveType::kNextNoDup: {
-            return cursor_->to_next_first_multi(throw_notfound);
+    try {
+        switch (type) {
+            case MoveType::kNext: {
+                return cursor_->to_next(throw_notfound);
+            }
+            case MoveType::kNextDup: {
+                return cursor_->to_current_next_multi(throw_notfound);
+            }
+            case MoveType::kNextNoDup: {
+                return cursor_->to_next_first_multi(throw_notfound);
+            }
+            default: {  // Avoid GCC complaining w/ error: control reaches end of non-void function
+                return CursorResult{{}, {}, false};
+            }
         }
-        default: {  // Avoid GCC complaining w/ error: control reaches end of non-void function
+    } catch (const mdbx::no_data& ex) {
+        // special case when cursor moves beyond existing data and is not handled by MDBX_NOTFOUND
+        if (!throw_notfound) {
             return CursorResult{{}, {}, false};
         }
+        throw ex;
     }
 }
 

silkworm/db/kv/api/local_cursor.cpp

@@ -11,6 +11,7 @@
 namespace silkworm::db::kv::api {
 
 using namespace silkworm::datastore::kvdb;
+using datastore::kvdb::detail::slice_as_bytes;
 using datastore::kvdb::detail::slice_as_hex;
 
 Task<void> LocalCursor::open_cursor(std::string_view table_name, bool is_dup_sorted) {
@@ -33,7 +34,7 @@ Task<KeyValue> LocalCursor::seek(ByteView key) {
 
     if (result) {
         SILK_DEBUG << "LocalCursor::seek found: key: " << key << " value: " << slice_as_hex(result.value);
-        co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())};
+        co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)};
     } else {
         SILK_DEBUG << "LocalCursor::seek not found key: " << key;
         co_return KeyValue{};
@@ -49,7 +50,7 @@ Task<KeyValue> LocalCursor::seek_exact(ByteView key) {
         SILK_DEBUG << "LocalCursor::seek_exact result: " << detail::dump_mdbx_result(result);
         if (result) {
             SILK_DEBUG << "LocalCursor::seek_exact found: key: " << key << " value: " << slice_as_hex(result.value);
-            co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())};
+            co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)};
         }
         SILK_ERROR << "LocalCursor::seek_exact !result key: " << key;  // TODO(canepat) handle properly?
     }
@@ -66,7 +67,7 @@ Task<KeyValue> LocalCursor::first() {
     }
 
     SILK_DEBUG << "LocalCursor::first: key: " << slice_as_hex(result.key) << " value: " << slice_as_hex(result.value);
-    co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())};
+    co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)};
 }
 
 Task<KeyValue> LocalCursor::last() {
@@ -79,7 +80,7 @@ Task<KeyValue> LocalCursor::last() {
     }
 
     SILK_DEBUG << "LocalCursor::last: key: " << slice_as_hex(result.key) << " value: " << slice_as_hex(result.value);
-    co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())};
+    co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)};
 }
 
 Task<KeyValue> LocalCursor::next() {
@@ -91,7 +92,7 @@ Task<KeyValue> LocalCursor::next() {
     if (result) {
         SILK_DEBUG << "LocalCursor::next: "
                    << " key: " << string_view_to_byte_view(result.key.as_string()) << " value: " << string_view_to_byte_view(result.value.as_string());
-        co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())};
+        co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)};
     } else {
         SILK_ERROR << "LocalCursor::next !result";  // TODO(canepat) handle properly?
     }
@@ -107,7 +108,7 @@ Task<KeyValue> LocalCursor::previous() {
     if (result) {
         SILK_DEBUG << "LocalCursor::previous: "
                    << " key: " << string_view_to_byte_view(result.key.as_string()) << " value: " << string_view_to_byte_view(result.value.as_string());
-        co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())};
+        co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)};
     } else {
         SILK_ERROR << "LocalCursor::previous !result";  // TODO(canepat) handle properly?
     }
@@ -123,7 +124,7 @@ Task<KeyValue> LocalCursor::next_dup() {
     if (result) {
         SILK_DEBUG << "LocalCursor::next_dup: "
                    << " key: " << string_view_to_byte_view(result.key.as_string()) << " value: " << string_view_to_byte_view(result.value.as_string());
-        co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())};
+        co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)};
     } else {
         SILK_ERROR << "LocalCursor::next_dup !result";  // TODO(canepat) handle properly?
     }
@@ -138,9 +139,9 @@ Task<Bytes> LocalCursor::seek_both(ByteView key, ByteView value) {
 
     if (result) {
         SILK_DEBUG << "LocalCursor::seek_both key: " << string_view_to_byte_view(result.key.as_string()) << " value: " << string_view_to_byte_view(result.value.as_string());
-        co_return string_to_bytes(result.value.as_string());
+        co_return slice_as_bytes(result.value);
     }
-    co_return string_to_bytes("");
+    co_return Bytes{};
 }
 
 Task<KeyValue> LocalCursor::seek_both_exact(ByteView key, ByteView value) {
@@ -152,7 +153,7 @@ Task<KeyValue> LocalCursor::seek_both_exact(ByteView key, ByteView value) {
     if (result) {
         SILK_DEBUG << "LocalCursor::seek_both_exact: "
                    << " key: " << string_view_to_byte_view(result.key.as_string()) << " value: " << string_view_to_byte_view(result.value.as_string());
-        co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())};
+        co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)};
     } else {
         SILK_ERROR << "LocalCursor::seek_both_exact !found key: " << key << " subkey:" << value;
     }

silkworm/db/kv/grpc/server/kv_calls.cpp

@@ -24,6 +24,7 @@ using boost::asio::as_tuple;
 using namespace boost::asio::experimental::awaitable_operators;
 using boost::asio::steady_timer;
 using boost::asio::use_awaitable;
+using datastore::kvdb::detail::slice_as_string;
 
 api::Version higher_version_ignoring_patch(api::Version lhs, api::Version rhs) {
     uint32_t lhs_major = std::get<0>(lhs);
@@ -350,7 +351,7 @@ bool TxCall::save_cursors(std::vector<CursorPosition>& positions) {
             }
             mdbx::slice key = result.key;
             mdbx::slice value = result.value;
-            positions.emplace_back(CursorPosition{key.as_string(), value.as_string()});
+            positions.emplace_back(CursorPosition{slice_as_string(key), slice_as_string(value)});
         }
     }