diff --git a/.circleci/config.yml b/.circleci/config.yml index c6406671fe..68051b3256 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -95,7 +95,7 @@ commands: command: | if [[ "<>" == "clang" ]] then - conan_profile=linux_x64_clang_16_release + conan_profile=linux_x64_clang_18_release else conan_profile=linux_x64_gcc_11_release fi @@ -132,7 +132,7 @@ commands: - run: name: "CMake Fuzzer" working_directory: ~/build - command: cmake ../project -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_16_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON + command: cmake ../project -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_18_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON - run: name: "Build Fuzzer" command: | @@ -256,7 +256,7 @@ jobs: - build_using_conan: build_type: Debug compiler_id: clang - compiler_version: 17 + compiler_version: 18 - test: sanitizer: address,undefined @@ -415,7 +415,7 @@ jobs: parameters: clang_version_min: type: integer - default: 16 + default: 18 clang_version_latest: type: integer default: 18 diff --git a/.github/actions/fuzzer-common-steps/action.yml b/.github/actions/fuzzer-common-steps/action.yml index d524d19e52..d8aa868b17 100644 --- a/.github/actions/fuzzer-common-steps/action.yml +++ b/.github/actions/fuzzer-common-steps/action.yml @@ -11,7 +11,7 @@ runs: steps: - name: Install Compiler shell: bash - run: cmake/setup/compiler_install.sh clang 16 + run: cmake/setup/compiler_install.sh clang 18 - name: Clean Build Directory shell: bash @@ -28,7 +28,7 @@ runs: - name: Configure CMake shell: bash working-directory: ${{runner.workspace}}/silkworm/build - run: CC=clang-16 CXX=clang++-16 cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_16_release -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON -DSILKWORM_FUZZER_SANITIZERS=${{inputs.fuzzer_sanitizers}} + run: CC=clang-16 CXX=clang++-16 cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_18_release -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON -DSILKWORM_FUZZER_SANITIZERS=${{inputs.fuzzer_sanitizers}} - name: Build Silkworm Fuzzer Test shell: bash diff --git a/cmake/conan_quiet.cmake b/cmake/conan_quiet.cmake index 10a8ab951c..70bc6168f9 100644 --- a/cmake/conan_quiet.cmake +++ b/cmake/conan_quiet.cmake @@ -8,7 +8,6 @@ set(GTest_FIND_QUIETLY YES) set(LibtorrentRasterbar_FIND_QUIETLY YES) set(Snappy_FIND_QUIETLY YES) set(Microsoft.GSL_FIND_QUIETLY YES) -set(OpenSSL_FIND_QUIETLY YES) set(fmt_FIND_QUIETLY YES) set(roaring_FIND_QUIETLY YES) set(BZip2_FIND_QUIETLY YES) @@ -25,15 +24,11 @@ set(asio-grpc_FIND_QUIETLY YES) set(benchmark_FIND_QUIETLY YES) set(gmp_FIND_QUIETLY YES) set(Boost_FIND_QUIETLY YES) -set(SQLite3_FIND_QUIETLY YES) set(mimalloc_FIND_QUIETLY YES) set(re2_FIND_QUIETLY YES) set(gRPC_FIND_QUIETLY YES) set(protobuf_FIND_QUIETLY YES) -set(ZLIB_FIND_QUIETLY YES) set(Protobuf_FIND_QUIETLY YES) -set(BZip2_FIND_QUIETLY YES) set(SQLiteCpp_FIND_QUIETLY YES) -set(GTest_FIND_QUIETLY YES) set(nlohmann_json_FIND_QUIETLY YES) set(libdeflate_FIND_QUIETLY YES) diff --git a/cmake/profiles/linux_x64_clang_16_debug b/cmake/profiles/linux_x64_clang_18_debug similarity index 84% rename from cmake/profiles/linux_x64_clang_16_debug rename to cmake/profiles/linux_x64_clang_18_debug index aadab8042d..7095870b39 100644 --- a/cmake/profiles/linux_x64_clang_16_debug +++ b/cmake/profiles/linux_x64_clang_18_debug @@ -2,7 +2,7 @@ os=Linux arch=x86_64 compiler=clang -compiler.version=16 +compiler.version=18 compiler.libcxx=libc++ compiler.cppstd=17 build_type=Debug diff --git a/cmake/profiles/linux_x64_clang_16_release b/cmake/profiles/linux_x64_clang_18_release similarity index 84% rename from cmake/profiles/linux_x64_clang_16_release rename to cmake/profiles/linux_x64_clang_18_release index 3164500a5a..4db13e30f0 100644 --- a/cmake/profiles/linux_x64_clang_16_release +++ b/cmake/profiles/linux_x64_clang_18_release @@ -2,7 +2,7 @@ os=Linux arch=x86_64 compiler=clang -compiler.version=16 +compiler.version=18 compiler.libcxx=libc++ compiler.cppstd=17 build_type=Release diff --git a/docs/fuzzer.md b/docs/fuzzer.md index 8a8553ce65..e5f284ba3f 100644 --- a/docs/fuzzer.md +++ b/docs/fuzzer.md @@ -6,7 +6,7 @@ To build the fuzzer use the following: ```bash mkdir build cd build -cmake ../project -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_16_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON +cmake ../project -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_18_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON cmake --build --target rpcdaemon_fuzzer_test cmake –-build --target rpcdaemon_fuzzer_diagnostics ``` diff --git a/silkworm/db/datastore/kvdb/mdbx.cpp b/silkworm/db/datastore/kvdb/mdbx.cpp index 6535de13ca..fdcda05348 100644 --- a/silkworm/db/datastore/kvdb/mdbx.cpp +++ b/silkworm/db/datastore/kvdb/mdbx.cpp @@ -21,7 +21,15 @@ namespace detail { } std::string slice_as_hex(const Slice& data) { - return ::mdbx::to_hex(data).as_string(); + return std::string(::mdbx::to_hex(data).as_string()); + } + + std::string slice_as_string(const Slice& data) { + return std::string(data.as_string()); + } + + silkworm::Bytes slice_as_bytes(const Slice& data) { + return silkworm::Bytes{reinterpret_cast(data.data()), data.size()}; } log::Args log_args_for_commit_latency(const MDBX_commit_latency& commit_latency) { @@ -106,7 +114,7 @@ ::mdbx::env_managed open_env(const EnvConfig& config) { throw std::runtime_error("Database map size is too small. Min required " + human_size(db_file_size)); } - uint32_t flags{MDBX_NOTLS | MDBX_NORDAHEAD | MDBX_COALESCE | MDBX_SYNC_DURABLE}; // Default flags + uint32_t flags{MDBX_NORDAHEAD | MDBX_SYNC_DURABLE}; // Default flags if (config.read_ahead) { flags &= ~MDBX_NORDAHEAD; @@ -133,6 +141,9 @@ ::mdbx::env_managed open_env(const EnvConfig& config) { if (config.shared) { flags |= MDBX_ACCEDE; } + if (config.no_sticky_threads) { + flags |= MDBX_NOSTICKYTHREADS; + } if (config.write_map) { flags |= MDBX_WRITEMAP; } @@ -574,7 +585,7 @@ std::vector list_maps(::mdbx::txn& tx, bool throw_notfound) { ::mdbx::map_handle main_map{1}; auto main_cursor{tx.open_cursor(main_map)}; for (auto it{main_cursor.to_first(throw_notfound)}; it.done; it = main_cursor.to_next(throw_notfound)) { - map_names.push_back(it.key.as_string()); + map_names.push_back(detail::slice_as_string(it.key)); } return map_names; } diff --git a/silkworm/db/datastore/kvdb/mdbx.hpp b/silkworm/db/datastore/kvdb/mdbx.hpp index a19494867e..14b343cfb6 100644 --- a/silkworm/db/datastore/kvdb/mdbx.hpp +++ b/silkworm/db/datastore/kvdb/mdbx.hpp @@ -60,6 +60,8 @@ namespace detail { std::string dump_mdbx_result(const CursorResult& result); std::string slice_as_hex(const Slice& data); + std::string slice_as_string(const Slice& data); + silkworm::Bytes slice_as_bytes(const Slice& data); } // namespace detail class ROTxn; @@ -367,6 +369,7 @@ struct EnvConfig { bool in_memory{false}; // Whether this db is in memory bool shared{false}; // Whether this process opens a db already opened by another process bool read_ahead{false}; // Whether to enable mdbx read ahead + bool no_sticky_threads{false}; // Whether to allow moving db transactions between threads bool write_map{false}; // Whether to enable mdbx write map size_t page_size{os::page_size()}; // Mdbx page size size_t max_size{3_Tebi}; // Mdbx max map size diff --git a/silkworm/db/datastore/kvdb/memory_mutation_cursor.cpp b/silkworm/db/datastore/kvdb/memory_mutation_cursor.cpp index 734e4151d3..790777564a 100644 --- a/silkworm/db/datastore/kvdb/memory_mutation_cursor.cpp +++ b/silkworm/db/datastore/kvdb/memory_mutation_cursor.cpp @@ -523,6 +523,14 @@ CursorResult MemoryMutationCursor::to_current_next_multi() { } CursorResult MemoryMutationCursor::to_current_next_multi(bool throw_notfound) { + if (config_.value_mode == ::mdbx::value_mode::single) { + if (throw_notfound) { + throw_error_notfound(); + } else { + return CursorResult{{}, {}, false}; + } + } + if (is_table_cleared()) { return memory_cursor_->to_current_next_multi(throw_notfound); } @@ -598,7 +606,7 @@ CursorResult MemoryMutationCursor::find_multivalue(const Slice& key, const Slice const auto memory_result = memory_cursor_->find_multivalue(key, value, false); auto db_result = cursor_->find_multivalue(key, value, false); - if (db_result.key && is_entry_deleted(db_result.key, db_result.value)) { + if (db_result.done && is_entry_deleted(db_result.key, db_result.value)) { db_result = next_on_db(MoveType::kNextDup, throw_notfound); } @@ -702,7 +710,7 @@ CursorResult MemoryMutationCursor::next_on_db(MemoryMutationCursor::MoveType typ CursorResult result = next_by_type(type, throw_notfound); if (!result.done) return result; - while (result.key && result.value && is_entry_deleted(result.key, result.value)) { + while (result.done && result.value && is_entry_deleted(result.key, result.value)) { result = next_by_type(type, throw_notfound); if (!result.done) return result; } @@ -711,19 +719,27 @@ CursorResult MemoryMutationCursor::next_on_db(MemoryMutationCursor::MoveType typ } CursorResult MemoryMutationCursor::next_by_type(MemoryMutationCursor::MoveType type, bool throw_notfound) { - switch (type) { - case MoveType::kNext: { - return cursor_->to_next(throw_notfound); - } - case MoveType::kNextDup: { - return cursor_->to_current_next_multi(throw_notfound); - } - case MoveType::kNextNoDup: { - return cursor_->to_next_first_multi(throw_notfound); + try { + switch (type) { + case MoveType::kNext: { + return cursor_->to_next(throw_notfound); + } + case MoveType::kNextDup: { + return cursor_->to_current_next_multi(throw_notfound); + } + case MoveType::kNextNoDup: { + return cursor_->to_next_first_multi(throw_notfound); + } + default: { // Avoid GCC complaining w/ error: control reaches end of non-void function + return CursorResult{{}, {}, false}; + } } - default: { // Avoid GCC complaining w/ error: control reaches end of non-void function + } catch (const mdbx::no_data& ex) { + // special case when cursor moves beyond existing data and is not handled by MDBX_NOTFOUND + if (!throw_notfound) { return CursorResult{{}, {}, false}; } + throw ex; } } diff --git a/silkworm/db/kv/api/local_cursor.cpp b/silkworm/db/kv/api/local_cursor.cpp index 5e07157153..9e3feea460 100644 --- a/silkworm/db/kv/api/local_cursor.cpp +++ b/silkworm/db/kv/api/local_cursor.cpp @@ -11,6 +11,7 @@ namespace silkworm::db::kv::api { using namespace silkworm::datastore::kvdb; +using datastore::kvdb::detail::slice_as_bytes; using datastore::kvdb::detail::slice_as_hex; Task LocalCursor::open_cursor(std::string_view table_name, bool is_dup_sorted) { @@ -33,7 +34,7 @@ Task LocalCursor::seek(ByteView key) { if (result) { SILK_DEBUG << "LocalCursor::seek found: key: " << key << " value: " << slice_as_hex(result.value); - co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())}; + co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)}; } else { SILK_DEBUG << "LocalCursor::seek not found key: " << key; co_return KeyValue{}; @@ -49,7 +50,7 @@ Task LocalCursor::seek_exact(ByteView key) { SILK_DEBUG << "LocalCursor::seek_exact result: " << detail::dump_mdbx_result(result); if (result) { SILK_DEBUG << "LocalCursor::seek_exact found: key: " << key << " value: " << slice_as_hex(result.value); - co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())}; + co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)}; } SILK_ERROR << "LocalCursor::seek_exact !result key: " << key; // TODO(canepat) handle properly? } @@ -66,7 +67,7 @@ Task LocalCursor::first() { } SILK_DEBUG << "LocalCursor::first: key: " << slice_as_hex(result.key) << " value: " << slice_as_hex(result.value); - co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())}; + co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)}; } Task LocalCursor::last() { @@ -79,7 +80,7 @@ Task LocalCursor::last() { } SILK_DEBUG << "LocalCursor::last: key: " << slice_as_hex(result.key) << " value: " << slice_as_hex(result.value); - co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())}; + co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)}; } Task LocalCursor::next() { @@ -91,7 +92,7 @@ Task LocalCursor::next() { if (result) { SILK_DEBUG << "LocalCursor::next: " << " key: " << string_view_to_byte_view(result.key.as_string()) << " value: " << string_view_to_byte_view(result.value.as_string()); - co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())}; + co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)}; } else { SILK_ERROR << "LocalCursor::next !result"; // TODO(canepat) handle properly? } @@ -107,7 +108,7 @@ Task LocalCursor::previous() { if (result) { SILK_DEBUG << "LocalCursor::previous: " << " key: " << string_view_to_byte_view(result.key.as_string()) << " value: " << string_view_to_byte_view(result.value.as_string()); - co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())}; + co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)}; } else { SILK_ERROR << "LocalCursor::previous !result"; // TODO(canepat) handle properly? } @@ -123,7 +124,7 @@ Task LocalCursor::next_dup() { if (result) { SILK_DEBUG << "LocalCursor::next_dup: " << " key: " << string_view_to_byte_view(result.key.as_string()) << " value: " << string_view_to_byte_view(result.value.as_string()); - co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())}; + co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)}; } else { SILK_ERROR << "LocalCursor::next_dup !result"; // TODO(canepat) handle properly? } @@ -138,9 +139,9 @@ Task LocalCursor::seek_both(ByteView key, ByteView value) { if (result) { SILK_DEBUG << "LocalCursor::seek_both key: " << string_view_to_byte_view(result.key.as_string()) << " value: " << string_view_to_byte_view(result.value.as_string()); - co_return string_to_bytes(result.value.as_string()); + co_return slice_as_bytes(result.value); } - co_return string_to_bytes(""); + co_return Bytes{}; } Task LocalCursor::seek_both_exact(ByteView key, ByteView value) { @@ -152,7 +153,7 @@ Task LocalCursor::seek_both_exact(ByteView key, ByteView value) { if (result) { SILK_DEBUG << "LocalCursor::seek_both_exact: " << " key: " << string_view_to_byte_view(result.key.as_string()) << " value: " << string_view_to_byte_view(result.value.as_string()); - co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())}; + co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)}; } else { SILK_ERROR << "LocalCursor::seek_both_exact !found key: " << key << " subkey:" << value; } diff --git a/silkworm/db/kv/grpc/server/kv_calls.cpp b/silkworm/db/kv/grpc/server/kv_calls.cpp index 52c57bc4d0..461d1d312c 100644 --- a/silkworm/db/kv/grpc/server/kv_calls.cpp +++ b/silkworm/db/kv/grpc/server/kv_calls.cpp @@ -24,6 +24,7 @@ using boost::asio::as_tuple; using namespace boost::asio::experimental::awaitable_operators; using boost::asio::steady_timer; using boost::asio::use_awaitable; +using datastore::kvdb::detail::slice_as_string; api::Version higher_version_ignoring_patch(api::Version lhs, api::Version rhs) { uint32_t lhs_major = std::get<0>(lhs); @@ -350,7 +351,7 @@ bool TxCall::save_cursors(std::vector& positions) { } mdbx::slice key = result.key; mdbx::slice value = result.value; - positions.emplace_back(CursorPosition{key.as_string(), value.as_string()}); + positions.emplace_back(CursorPosition{slice_as_string(key), slice_as_string(value)}); } } diff --git a/silkworm/db/kv/grpc/server/kv_server_test.cpp b/silkworm/db/kv/grpc/server/kv_server_test.cpp index 81077c7789..3eba196bce 100644 --- a/silkworm/db/kv/grpc/server/kv_server_test.cpp +++ b/silkworm/db/kv/grpc/server/kv_server_test.cpp @@ -210,6 +210,7 @@ struct KvEnd2EndTest { db_config->path = data_dir.chaindata().path().string(); db_config->create = true; db_config->in_memory = true; + db_config->no_sticky_threads = true; database_env = datastore::kvdb::open_env(*db_config); auto rw_txn{database_env.start_write()}; datastore::kvdb::open_map(rw_txn, kTestMap); @@ -1004,7 +1005,7 @@ TEST_CASE("KvServer E2E: Tx cursor valid operations", "[silkworm][node][rpc]") { open1.set_op(remote::Op::OPEN); open1.set_bucket_name(kTestMap.name_str()); remote::Cursor next1; - next1.set_op(remote::Op::NEXT); + next1.set_op(remote::Op::FIRST); next1.set_cursor(0); // automatically assigned by KvClient::tx remote::Cursor close1; close1.set_op(remote::Op::CLOSE); @@ -1013,7 +1014,7 @@ TEST_CASE("KvServer E2E: Tx cursor valid operations", "[silkworm][node][rpc]") { open2.set_op(remote::Op::OPEN); open2.set_bucket_name(kTestMap.name_str()); remote::Cursor next2; - next2.set_op(remote::Op::NEXT); + next2.set_op(remote::Op::FIRST); next2.set_cursor(0); // automatically assigned by KvClient::tx remote::Cursor close2; close2.set_op(remote::Op::CLOSE); @@ -1091,7 +1092,7 @@ TEST_CASE("KvServer E2E: Tx cursor valid operations", "[silkworm][node][rpc]") { open1.set_op(remote::Op::OPEN); open1.set_bucket_name(kTestMap.name_str()); remote::Cursor prev1; - prev1.set_op(remote::Op::PREV); + prev1.set_op(remote::Op::LAST); prev1.set_cursor(0); // automatically assigned by KvClient::tx remote::Cursor close1; close1.set_op(remote::Op::CLOSE); @@ -1100,7 +1101,7 @@ TEST_CASE("KvServer E2E: Tx cursor valid operations", "[silkworm][node][rpc]") { open2.set_op(remote::Op::OPEN); open2.set_bucket_name(kTestMap.name_str()); remote::Cursor prev2; - prev2.set_op(remote::Op::PREV); + prev2.set_op(remote::Op::LAST); prev2.set_cursor(0); // automatically assigned by KvClient::tx remote::Cursor close2; close2.set_op(remote::Op::CLOSE); @@ -1803,22 +1804,19 @@ TEST_CASE("KvServer E2E: Tx cursor invalid operations", "[silkworm][node][rpc]") remote::Cursor open; open.set_op(remote::Op::OPEN); open.set_bucket_name(kTestMap.name_str()); - remote::Cursor first; - first.set_op(remote::Op::FIRST); - first.set_cursor(0); // automatically assigned by KvClient::tx remote::Cursor first_dup; first_dup.set_op(remote::Op::FIRST_DUP); first_dup.set_cursor(0); // automatically assigned by KvClient::tx remote::Cursor close; close.set_op(remote::Op::CLOSE); close.set_cursor(0); // automatically assigned by KvClient::tx - std::vector requests{open, first, first_dup, close}; + std::vector requests{open, first_dup, close}; std::vector responses; const auto status = kv_client.tx(requests, responses); CHECK(!status.ok()); CHECK(status.error_code() == ::grpc::StatusCode::INTERNAL); - CHECK(absl::StrContains(status.error_message(), "exception: MDBX_INCOMPATIBLE")); - CHECK(responses.size() == 3); + CHECK(absl::StrContains(status.error_message(), "exception: MDBX_ENODATA")); + CHECK(responses.size() == 2); CHECK(responses[0].tx_id() != 0); CHECK(responses[1].cursor_id() != 0); } @@ -1827,22 +1825,19 @@ TEST_CASE("KvServer E2E: Tx cursor invalid operations", "[silkworm][node][rpc]") remote::Cursor open; open.set_op(remote::Op::OPEN); open.set_bucket_name(kTestMap.name_str()); - remote::Cursor first; - first.set_op(remote::Op::FIRST); - first.set_cursor(0); // automatically assigned by KvClient::tx remote::Cursor last_dup; last_dup.set_op(remote::Op::LAST_DUP); last_dup.set_cursor(0); // automatically assigned by KvClient::tx remote::Cursor close; close.set_op(remote::Op::CLOSE); close.set_cursor(0); // automatically assigned by KvClient::tx - std::vector requests{open, first, last_dup, close}; + std::vector requests{open, last_dup, close}; std::vector responses; const auto status = kv_client.tx(requests, responses); CHECK(!status.ok()); CHECK(status.error_code() == ::grpc::StatusCode::INTERNAL); - CHECK(absl::StrContains(status.error_message(), "exception: MDBX_INCOMPATIBLE")); - CHECK(responses.size() == 3); + CHECK(absl::StrContains(status.error_message(), "exception: MDBX_ENODATA")); + CHECK(responses.size() == 2); CHECK(responses[0].tx_id() != 0); CHECK(responses[1].cursor_id() != 0); } @@ -1862,7 +1857,7 @@ TEST_CASE("KvServer E2E: Tx cursor invalid operations", "[silkworm][node][rpc]") const auto status = kv_client.tx(requests, responses); CHECK(!status.ok()); CHECK(status.error_code() == ::grpc::StatusCode::INTERNAL); - CHECK(absl::StrContains(status.error_message(), "exception: mdbx")); + CHECK(absl::StrContains(status.error_message(), "exception: MDBX_ENODATA")); CHECK(responses.size() == 2); CHECK(responses[0].tx_id() != 0); } @@ -1882,7 +1877,7 @@ TEST_CASE("KvServer E2E: Tx cursor invalid operations", "[silkworm][node][rpc]") const auto status = kv_client.tx(requests, responses); CHECK(!status.ok()); CHECK(status.error_code() == ::grpc::StatusCode::INTERNAL); - CHECK(absl::StrContains(status.error_message(), "exception: mdbx")); + CHECK(absl::StrContains(status.error_message(), "exception: MDBX_ENODATA")); CHECK(responses.size() == 2); CHECK(responses[0].tx_id() != 0); } diff --git a/silkworm/db/mdbx_test.cpp b/silkworm/db/mdbx_test.cpp index a6d0466308..eda2b57563 100644 --- a/silkworm/db/mdbx_test.cpp +++ b/silkworm/db/mdbx_test.cpp @@ -404,9 +404,8 @@ TEST_CASE("Cursor walk") { REQUIRE(table_cursor.size() == kGeneticCode.size()); REQUIRE(table_cursor.empty() == false); - // Rebind cursor so its position is undefined - table_cursor.bind(txn, {kTableName}); - REQUIRE(table_cursor.eof() == true); + // Reset cursor so its position is at the begining + table_cursor.to_first(); // read entire table forward cursor_for_each(table_cursor, save_all_data_map); @@ -587,16 +586,17 @@ TEST_CASE("Cursor walk") { //! Compute the maximum free space in an empty MDBX database page //! \warning this may change in future versions of MDBX //! \details see `page_space` function in MDBX core.c -static size_t page_space(const mdbx::env& env) { +static size_t page_space(const mdbx::txn& txn) { constexpr size_t kPageHeaderSize{20}; // size of MDBX page header (PAGEHDRSZ) - const size_t db_page_size{env.get_pagesize()}; + const auto stats = txn.env().get_stat(txn); + const size_t db_page_size{stats.ms_psize}; return db_page_size - kPageHeaderSize; } static size_t max_multivalue_size_for_leaf_page(const mdbx::txn& txn) { constexpr size_t kDupSortNodes{2}; constexpr size_t kNodeHeaderSize{8}; // size of MDBX node header (NODESIZE) - return page_space(txn.env()) / kDupSortNodes - 2 * kNodeHeaderSize; + return page_space(txn) / kDupSortNodes - 2 * kNodeHeaderSize; } TEST_CASE("OF pages") { @@ -849,10 +849,9 @@ TEST_CASE("Multi-value erase+upsert w/ same value increases free pages") { // We need to split max multi-value data size between key and value constexpr size_t kKeySize{20}; // just to copycat account address size - const size_t max_non_initial_value_size{[&env]() { - auto ro_txn{env.start_read()}; - return max_multivalue_size_for_leaf_page(ro_txn); - }()}; + auto ro_txn{env.start_read()}; + auto max_non_initial_value_size = max_multivalue_size_for_leaf_page(ro_txn); + ro_txn.abort(); const size_t max_first_value_size{max_non_initial_value_size - kKeySize}; // we need to take key size into account once const Bytes key(kKeySize, '\0'); diff --git a/silkworm/db/memory_mutation_cursor_test.cpp b/silkworm/db/memory_mutation_cursor_test.cpp index 4902de379f..772f63ef45 100644 --- a/silkworm/db/memory_mutation_cursor_test.cpp +++ b/silkworm/db/memory_mutation_cursor_test.cpp @@ -432,24 +432,12 @@ TEST_CASE("MemoryMutationCursor: to_current_next_multi", "[silkworm][node][db][m SECTION(tag + ": to_current_next_multi on existent table w/ positioning: OK") { MemoryMutationCursor mutation_cursor1{test->mutation, table::kCode}; REQUIRE(mutation_cursor1.to_first()); - const auto result1 = mutation_cursor1.to_current_next_multi(); - CHECK(result1.done); - CHECK(result1.key == "BB"); - CHECK(result1.value == "11"); + const auto result1 = mutation_cursor1.to_current_next_multi(/*throw_notfound=*/false); + CHECK(!result1.done); MemoryMutationCursor mutation_cursor2{test->mutation, table::kCode}; REQUIRE(mutation_cursor2.to_first()); - const auto result2 = mutation_cursor2.to_current_next_multi(/*throw_notfound=*/true); - CHECK(result2.done); - CHECK(result2.key == "BB"); - CHECK(result2.value == "11"); - - MemoryMutationCursor mutation_cursor3{test->mutation, table::kCode}; - REQUIRE(mutation_cursor3.to_first()); - const auto result3 = mutation_cursor3.to_current_next_multi(/*throw_notfound=*/false); - CHECK(result3.done); - CHECK(result3.key == "BB"); - CHECK(result3.value == "11"); + CHECK_THROWS_AS(mutation_cursor2.to_current_next_multi(/*throw_notfound=*/true), mdbx::not_found); MemoryMutationCursor mutation_cursor4{test->mutation, table::kAccountChangeSet}; REQUIRE(mutation_cursor4.to_first()); @@ -477,14 +465,8 @@ TEST_CASE("MemoryMutationCursor: to_current_next_multi", "[silkworm][node][db][m MemoryMutationCursor mutation_cursor1{test->mutation, table::kCode}; REQUIRE(mutation_cursor1.to_first(/*throw_notfound=*/false)); const auto result1 = mutation_cursor1.to_current_next_multi(/*throw_notfound=*/false); - CHECK(result1.done); - CHECK(result1.key == "BB"); - CHECK(result1.value == "11"); - const auto result2 = mutation_cursor1.to_current_next_multi(/*throw_notfound=*/false); - CHECK(!result2.done); - REQUIRE(mutation_cursor1.to_last(/*throw_notfound=*/false)); - const auto result3 = mutation_cursor1.to_current_next_multi(/*throw_notfound=*/false); - CHECK(!result3.done); + CHECK(!result1.done); + CHECK_THROWS_AS(mutation_cursor1.to_current_next_multi(/*throw_notfound=*/true), mdbx::not_found); MemoryMutationCursor mutation_cursor2{test->mutation, table::kAccountChangeSet}; REQUIRE(mutation_cursor2.to_first(/*throw_notfound=*/false)); @@ -1198,19 +1180,19 @@ TEST_CASE("MemoryMutationCursor: SeekBothRange db->mem->db->mem", "[silkworm][no TEST_CASE("MemoryMutationCursor: Delete db->mem->db->mem", "[silkworm][node][db][memory_mutation]") { MemoryMutationCursorTest test; - auto rw_db_cursor = test.main_txn.rw_cursor_dup_sort(table::kHashedAccounts); + auto rw_db_cursor = test.main_txn.rw_cursor(table::kHashedAccounts); rw_db_cursor->upsert(mdbx::slice{"key1"}, mdbx::slice{"value1.1"}); rw_db_cursor->upsert(mdbx::slice{"key3"}, mdbx::slice{"value3.3"}); test.main_txn.commit_and_renew(); - auto rw_mem_cursor = test.mutation.rw_cursor_dup_sort(table::kHashedAccounts); + auto rw_mem_cursor = test.mutation.rw_cursor(table::kHashedAccounts); rw_mem_cursor->upsert("key1", "value1.3"); rw_mem_cursor->upsert("key2", "value2.1"); rw_mem_cursor->upsert("key3", "value3.1"); test.mutation.commit_and_renew(); - auto db_cursor = test.main_txn.rw_cursor_dup_sort(table::kHashedAccounts); - auto mem_cursor = test.mutation.rw_cursor_dup_sort(table::kHashedAccounts); + auto db_cursor = test.main_txn.rw_cursor(table::kHashedAccounts); + auto mem_cursor = test.mutation.rw_cursor(table::kHashedAccounts); mem_cursor->erase(mdbx::slice{"key1"}); mem_cursor->erase(mdbx::slice{"key3"}); diff --git a/silkworm/db/prune_mode.cpp b/silkworm/db/prune_mode.cpp index a22a175c8d..779b7734b4 100644 --- a/silkworm/db/prune_mode.cpp +++ b/silkworm/db/prune_mode.cpp @@ -36,7 +36,7 @@ static BlockAmount read_block_amount_for_key(mdbx::cursor& source, std::string_v value_type = BlockAmount::Type::kBefore; } else { // Something bad has been written - throw std::runtime_error("Invalid prune type stored in database : " + data2.value.as_string()); + throw std::runtime_error("Invalid prune type stored in database : " + std::string(data2.value.as_string())); } } return BlockAmount(value_type, value); diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index 3e6284645c..3a5b6028c8 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -18,11 +18,11 @@ RUN git submodule sync RUN git submodule update --init --recursive # 3 Install compiler -RUN cmake/setup/compiler_install.sh clang 16 +RUN cmake/setup/compiler_install.sh clang 18 # 4 Build all targets WORKDIR /app/build -RUN cmake ../project -DCMAKE_BUILD_TYPE=Debug -DCONAN_PROFILE=linux_x64_clang_16_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake +RUN cmake ../project -DCMAKE_BUILD_TYPE=Debug -DCONAN_PROFILE=linux_x64_clang_18_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake WORKDIR /app/project RUN cmake --build /app/build -j4 @@ -48,7 +48,7 @@ RUN git pull origin ${BRANCH} # Rebuild all targets RUN echo "Rebuilding" WORKDIR /app/build -RUN sudo cmake ../project -DCMAKE_BUILD_TYPE=Debug -DCONAN_PROFILE=linux_x64_clang_16_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake +RUN sudo cmake ../project -DCMAKE_BUILD_TYPE=Debug -DCONAN_PROFILE=linux_x64_clang_18_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake WORKDIR /app/project RUN sudo cmake --build /app/build -j4 @@ -64,7 +64,7 @@ RUN sudo cmake --build /app/build -j4 # RUN for file in ~/artifacts/*; do sed -i 's/^>> //' "$file"; done # # Rebuild fuzzer # WORKDIR /app/build -# RUN sudo cmake ../project -DCMAKE_BUILD_TYPE=Debug -DCONAN_PROFILE=linux_x64_clang_16_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON +# RUN sudo cmake ../project -DCMAKE_BUILD_TYPE=Debug -DCONAN_PROFILE=linux_x64_clang_18_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON # WORKDIR /app/project # RUN sudo cmake --build /app/build -j4 --target rpcdaemon_fuzzer_test # # Run fuzzer diff --git a/third_party/erigon-mdbx-go/CMakeLists.txt b/third_party/erigon-mdbx-go/CMakeLists.txt index 161093fb84..f6a6c02104 100644 --- a/third_party/erigon-mdbx-go/CMakeLists.txt +++ b/third_party/erigon-mdbx-go/CMakeLists.txt @@ -9,7 +9,7 @@ if(NOT CMAKE_MESSAGE_LOG_LEVEL) set(CMAKE_MESSAGE_LOG_LEVEL NOTICE) endif() -add_subdirectory(mdbx-go/mdbxdist) +add_subdirectory(mdbx-go/libmdbx) # restore the log verbosity if(CMAKE_MESSAGE_LOG_LEVEL_EMPTY) diff --git a/third_party/erigon-mdbx-go/mdbx-go b/third_party/erigon-mdbx-go/mdbx-go index aab789d157..38a69c709d 160000 --- a/third_party/erigon-mdbx-go/mdbx-go +++ b/third_party/erigon-mdbx-go/mdbx-go @@ -1 +1 @@ -Subproject commit aab789d15716460c36950ddb3b9f3b7a7c4c8089 +Subproject commit 38a69c709de7d41c809bd20eebcc5f6ad4572c36