From e0b86d3709646198a7cf775c587030e122ee06d4 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Wed, 26 Mar 2025 08:13:57 +0100 Subject: [PATCH 01/10] Update libmdbx --- third_party/erigon-mdbx-go/CMakeLists.txt | 2 +- third_party/erigon-mdbx-go/mdbx-go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/third_party/erigon-mdbx-go/CMakeLists.txt b/third_party/erigon-mdbx-go/CMakeLists.txt index 044635037e..d3c2106c60 100644 --- a/third_party/erigon-mdbx-go/CMakeLists.txt +++ b/third_party/erigon-mdbx-go/CMakeLists.txt @@ -22,7 +22,7 @@ if(NOT CMAKE_MESSAGE_LOG_LEVEL) set(CMAKE_MESSAGE_LOG_LEVEL NOTICE) endif() -add_subdirectory(mdbx-go/mdbxdist) +add_subdirectory(mdbx-go/libmdbx) # restore the log verbosity if(CMAKE_MESSAGE_LOG_LEVEL_EMPTY) diff --git a/third_party/erigon-mdbx-go/mdbx-go b/third_party/erigon-mdbx-go/mdbx-go index aab789d157..38a69c709d 160000 --- a/third_party/erigon-mdbx-go/mdbx-go +++ b/third_party/erigon-mdbx-go/mdbx-go @@ -1 +1 @@ -Subproject commit aab789d15716460c36950ddb3b9f3b7a7c4c8089 +Subproject commit 38a69c709de7d41c809bd20eebcc5f6ad4572c36 From 5b9e286005898bd9e14c3c9e701a3967c7baecfa Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Tue, 1 Apr 2025 13:44:21 +0200 Subject: [PATCH 02/10] Update compiler version --- .circleci/config.yml | 4 ++-- cmake/conan_quiet.cmake | 5 ----- cmake/profiles/linux_x64_clang_16_debug | 2 +- silkworm/db/datastore/kvdb/mdbx.cpp | 2 +- 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 03cf83018a..2b69e4b963 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -270,7 +270,7 @@ jobs: - build_using_conan: build_type: Debug compiler_id: clang - compiler_version: 17 + compiler_version: 18 - test: sanitizer: address,undefined @@ -429,7 +429,7 @@ jobs: parameters: clang_version_min: type: integer - default: 16 + default: 18 clang_version_latest: type: integer default: 18 diff --git a/cmake/conan_quiet.cmake b/cmake/conan_quiet.cmake index 6a3bacc593..f8953ccddf 100644 --- a/cmake/conan_quiet.cmake +++ b/cmake/conan_quiet.cmake @@ -8,7 +8,6 @@ set(GTest_FIND_QUIETLY YES) set(LibtorrentRasterbar_FIND_QUIETLY YES) set(Snappy_FIND_QUIETLY YES) set(Microsoft.GSL_FIND_QUIETLY YES) -set(OpenSSL_FIND_QUIETLY YES) set(fmt_FIND_QUIETLY YES) set(roaring_FIND_QUIETLY YES) set(BZip2_FIND_QUIETLY YES) @@ -25,14 +24,10 @@ set(asio-grpc_FIND_QUIETLY YES) set(benchmark_FIND_QUIETLY YES) set(gmp_FIND_QUIETLY YES) set(Boost_FIND_QUIETLY YES) -set(SQLite3_FIND_QUIETLY YES) set(mimalloc_FIND_QUIETLY YES) set(re2_FIND_QUIETLY YES) set(gRPC_FIND_QUIETLY YES) set(protobuf_FIND_QUIETLY YES) -set(ZLIB_FIND_QUIETLY YES) set(Protobuf_FIND_QUIETLY YES) -set(BZip2_FIND_QUIETLY YES) set(SQLiteCpp_FIND_QUIETLY YES) -set(GTest_FIND_QUIETLY YES) set(nlohmann_json_FIND_QUIETLY YES) diff --git a/cmake/profiles/linux_x64_clang_16_debug b/cmake/profiles/linux_x64_clang_16_debug index aadab8042d..7095870b39 100644 --- a/cmake/profiles/linux_x64_clang_16_debug +++ b/cmake/profiles/linux_x64_clang_16_debug @@ -2,7 +2,7 @@ os=Linux arch=x86_64 compiler=clang -compiler.version=16 +compiler.version=18 compiler.libcxx=libc++ compiler.cppstd=17 build_type=Debug diff --git a/silkworm/db/datastore/kvdb/mdbx.cpp b/silkworm/db/datastore/kvdb/mdbx.cpp index b458f24a53..179218c27c 100644 --- a/silkworm/db/datastore/kvdb/mdbx.cpp +++ b/silkworm/db/datastore/kvdb/mdbx.cpp @@ -119,7 +119,7 @@ ::mdbx::env_managed open_env(const EnvConfig& config) { throw std::runtime_error("Database map size is too small. Min required " + human_size(db_file_size)); } - uint32_t flags{MDBX_NOTLS | MDBX_NORDAHEAD | MDBX_COALESCE | MDBX_SYNC_DURABLE}; // Default flags + uint32_t flags{ MDBX_NORDAHEAD | MDBX_SYNC_DURABLE}; // Default flags if (config.read_ahead) { flags &= ~MDBX_NORDAHEAD; From 2b85f0305936a651ada5db7b6caec163e82e97ed Mon Sep 17 00:00:00 2001 From: GitHub Date: Tue, 1 Apr 2025 11:44:56 +0000 Subject: [PATCH 03/10] make fmt --- silkworm/db/datastore/kvdb/mdbx.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/silkworm/db/datastore/kvdb/mdbx.cpp b/silkworm/db/datastore/kvdb/mdbx.cpp index 179218c27c..91b3dbf30b 100644 --- a/silkworm/db/datastore/kvdb/mdbx.cpp +++ b/silkworm/db/datastore/kvdb/mdbx.cpp @@ -119,7 +119,7 @@ ::mdbx::env_managed open_env(const EnvConfig& config) { throw std::runtime_error("Database map size is too small. Min required " + human_size(db_file_size)); } - uint32_t flags{ MDBX_NORDAHEAD | MDBX_SYNC_DURABLE}; // Default flags + uint32_t flags{MDBX_NORDAHEAD | MDBX_SYNC_DURABLE}; // Default flags if (config.read_ahead) { flags &= ~MDBX_NORDAHEAD; From 13cc1b59438e95c1c635e23991043275bc4faa8d Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Wed, 23 Apr 2025 16:41:13 +0200 Subject: [PATCH 04/10] Fix tests for libmdbx 0.13 --- silkworm/db/datastore/kvdb/mdbx.cpp | 3 ++ silkworm/db/datastore/kvdb/mdbx.hpp | 1 + .../datastore/kvdb/memory_mutation_cursor.cpp | 40 +++++++++++++------ silkworm/db/kv/grpc/server/kv_server_test.cpp | 31 ++++++-------- silkworm/db/mdbx_test.cpp | 20 +++++----- silkworm/db/memory_mutation_cursor_test.cpp | 36 +++++------------ 6 files changed, 64 insertions(+), 67 deletions(-) diff --git a/silkworm/db/datastore/kvdb/mdbx.cpp b/silkworm/db/datastore/kvdb/mdbx.cpp index dffffdadc5..a18c6c0f68 100644 --- a/silkworm/db/datastore/kvdb/mdbx.cpp +++ b/silkworm/db/datastore/kvdb/mdbx.cpp @@ -133,6 +133,9 @@ ::mdbx::env_managed open_env(const EnvConfig& config) { if (config.shared) { flags |= MDBX_ACCEDE; } + if (config.no_sticky_threads) { + flags |= MDBX_NOSTICKYTHREADS; + } if (config.write_map) { flags |= MDBX_WRITEMAP; } diff --git a/silkworm/db/datastore/kvdb/mdbx.hpp b/silkworm/db/datastore/kvdb/mdbx.hpp index a19494867e..af6264562c 100644 --- a/silkworm/db/datastore/kvdb/mdbx.hpp +++ b/silkworm/db/datastore/kvdb/mdbx.hpp @@ -367,6 +367,7 @@ struct EnvConfig { bool in_memory{false}; // Whether this db is in memory bool shared{false}; // Whether this process opens a db already opened by another process bool read_ahead{false}; // Whether to enable mdbx read ahead + bool no_sticky_threads{false}; // Whether to allow moving db transactions between threads bool write_map{false}; // Whether to enable mdbx write map size_t page_size{os::page_size()}; // Mdbx page size size_t max_size{3_Tebi}; // Mdbx max map size diff --git a/silkworm/db/datastore/kvdb/memory_mutation_cursor.cpp b/silkworm/db/datastore/kvdb/memory_mutation_cursor.cpp index 734e4151d3..790777564a 100644 --- a/silkworm/db/datastore/kvdb/memory_mutation_cursor.cpp +++ b/silkworm/db/datastore/kvdb/memory_mutation_cursor.cpp @@ -523,6 +523,14 @@ CursorResult MemoryMutationCursor::to_current_next_multi() { } CursorResult MemoryMutationCursor::to_current_next_multi(bool throw_notfound) { + if (config_.value_mode == ::mdbx::value_mode::single) { + if (throw_notfound) { + throw_error_notfound(); + } else { + return CursorResult{{}, {}, false}; + } + } + if (is_table_cleared()) { return memory_cursor_->to_current_next_multi(throw_notfound); } @@ -598,7 +606,7 @@ CursorResult MemoryMutationCursor::find_multivalue(const Slice& key, const Slice const auto memory_result = memory_cursor_->find_multivalue(key, value, false); auto db_result = cursor_->find_multivalue(key, value, false); - if (db_result.key && is_entry_deleted(db_result.key, db_result.value)) { + if (db_result.done && is_entry_deleted(db_result.key, db_result.value)) { db_result = next_on_db(MoveType::kNextDup, throw_notfound); } @@ -702,7 +710,7 @@ CursorResult MemoryMutationCursor::next_on_db(MemoryMutationCursor::MoveType typ CursorResult result = next_by_type(type, throw_notfound); if (!result.done) return result; - while (result.key && result.value && is_entry_deleted(result.key, result.value)) { + while (result.done && result.value && is_entry_deleted(result.key, result.value)) { result = next_by_type(type, throw_notfound); if (!result.done) return result; } @@ -711,19 +719,27 @@ CursorResult MemoryMutationCursor::next_on_db(MemoryMutationCursor::MoveType typ } CursorResult MemoryMutationCursor::next_by_type(MemoryMutationCursor::MoveType type, bool throw_notfound) { - switch (type) { - case MoveType::kNext: { - return cursor_->to_next(throw_notfound); - } - case MoveType::kNextDup: { - return cursor_->to_current_next_multi(throw_notfound); - } - case MoveType::kNextNoDup: { - return cursor_->to_next_first_multi(throw_notfound); + try { + switch (type) { + case MoveType::kNext: { + return cursor_->to_next(throw_notfound); + } + case MoveType::kNextDup: { + return cursor_->to_current_next_multi(throw_notfound); + } + case MoveType::kNextNoDup: { + return cursor_->to_next_first_multi(throw_notfound); + } + default: { // Avoid GCC complaining w/ error: control reaches end of non-void function + return CursorResult{{}, {}, false}; + } } - default: { // Avoid GCC complaining w/ error: control reaches end of non-void function + } catch (const mdbx::no_data& ex) { + // special case when cursor moves beyond existing data and is not handled by MDBX_NOTFOUND + if (!throw_notfound) { return CursorResult{{}, {}, false}; } + throw ex; } } diff --git a/silkworm/db/kv/grpc/server/kv_server_test.cpp b/silkworm/db/kv/grpc/server/kv_server_test.cpp index b8932b585c..0c31ccee49 100644 --- a/silkworm/db/kv/grpc/server/kv_server_test.cpp +++ b/silkworm/db/kv/grpc/server/kv_server_test.cpp @@ -210,6 +210,7 @@ struct KvEnd2EndTest { db_config->path = data_dir.chaindata().path().string(); db_config->create = true; db_config->in_memory = true; + db_config->no_sticky_threads = true; database_env = datastore::kvdb::open_env(*db_config); auto rw_txn{database_env.start_write()}; datastore::kvdb::open_map(rw_txn, kTestMap); @@ -1004,7 +1005,7 @@ TEST_CASE("KvServer E2E: Tx cursor valid operations", "[silkworm][node][rpc]") { open1.set_op(remote::Op::OPEN); open1.set_bucket_name(kTestMap.name); remote::Cursor next1; - next1.set_op(remote::Op::NEXT); + next1.set_op(remote::Op::FIRST); next1.set_cursor(0); // automatically assigned by KvClient::tx remote::Cursor close1; close1.set_op(remote::Op::CLOSE); @@ -1013,7 +1014,7 @@ TEST_CASE("KvServer E2E: Tx cursor valid operations", "[silkworm][node][rpc]") { open2.set_op(remote::Op::OPEN); open2.set_bucket_name(kTestMap.name); remote::Cursor next2; - next2.set_op(remote::Op::NEXT); + next2.set_op(remote::Op::FIRST); next2.set_cursor(0); // automatically assigned by KvClient::tx remote::Cursor close2; close2.set_op(remote::Op::CLOSE); @@ -1091,7 +1092,7 @@ TEST_CASE("KvServer E2E: Tx cursor valid operations", "[silkworm][node][rpc]") { open1.set_op(remote::Op::OPEN); open1.set_bucket_name(kTestMap.name); remote::Cursor prev1; - prev1.set_op(remote::Op::PREV); + prev1.set_op(remote::Op::LAST); prev1.set_cursor(0); // automatically assigned by KvClient::tx remote::Cursor close1; close1.set_op(remote::Op::CLOSE); @@ -1100,7 +1101,7 @@ TEST_CASE("KvServer E2E: Tx cursor valid operations", "[silkworm][node][rpc]") { open2.set_op(remote::Op::OPEN); open2.set_bucket_name(kTestMap.name); remote::Cursor prev2; - prev2.set_op(remote::Op::PREV); + prev2.set_op(remote::Op::LAST); prev2.set_cursor(0); // automatically assigned by KvClient::tx remote::Cursor close2; close2.set_op(remote::Op::CLOSE); @@ -1803,22 +1804,19 @@ TEST_CASE("KvServer E2E: Tx cursor invalid operations", "[silkworm][node][rpc]") remote::Cursor open; open.set_op(remote::Op::OPEN); open.set_bucket_name(kTestMap.name); - remote::Cursor first; - first.set_op(remote::Op::FIRST); - first.set_cursor(0); // automatically assigned by KvClient::tx remote::Cursor first_dup; first_dup.set_op(remote::Op::FIRST_DUP); first_dup.set_cursor(0); // automatically assigned by KvClient::tx remote::Cursor close; close.set_op(remote::Op::CLOSE); close.set_cursor(0); // automatically assigned by KvClient::tx - std::vector requests{open, first, first_dup, close}; + std::vector requests{open, first_dup, close}; std::vector responses; const auto status = kv_client.tx(requests, responses); CHECK(!status.ok()); CHECK(status.error_code() == ::grpc::StatusCode::INTERNAL); - CHECK(absl::StrContains(status.error_message(), "exception: MDBX_INCOMPATIBLE")); - CHECK(responses.size() == 3); + CHECK(absl::StrContains(status.error_message(), "exception: MDBX_ENODATA")); + CHECK(responses.size() == 2); CHECK(responses[0].tx_id() != 0); CHECK(responses[1].cursor_id() != 0); } @@ -1827,22 +1825,19 @@ TEST_CASE("KvServer E2E: Tx cursor invalid operations", "[silkworm][node][rpc]") remote::Cursor open; open.set_op(remote::Op::OPEN); open.set_bucket_name(kTestMap.name); - remote::Cursor first; - first.set_op(remote::Op::FIRST); - first.set_cursor(0); // automatically assigned by KvClient::tx remote::Cursor last_dup; last_dup.set_op(remote::Op::LAST_DUP); last_dup.set_cursor(0); // automatically assigned by KvClient::tx remote::Cursor close; close.set_op(remote::Op::CLOSE); close.set_cursor(0); // automatically assigned by KvClient::tx - std::vector requests{open, first, last_dup, close}; + std::vector requests{open, last_dup, close}; std::vector responses; const auto status = kv_client.tx(requests, responses); CHECK(!status.ok()); CHECK(status.error_code() == ::grpc::StatusCode::INTERNAL); - CHECK(absl::StrContains(status.error_message(), "exception: MDBX_INCOMPATIBLE")); - CHECK(responses.size() == 3); + CHECK(absl::StrContains(status.error_message(), "exception: MDBX_ENODATA")); + CHECK(responses.size() == 2); CHECK(responses[0].tx_id() != 0); CHECK(responses[1].cursor_id() != 0); } @@ -1862,7 +1857,7 @@ TEST_CASE("KvServer E2E: Tx cursor invalid operations", "[silkworm][node][rpc]") const auto status = kv_client.tx(requests, responses); CHECK(!status.ok()); CHECK(status.error_code() == ::grpc::StatusCode::INTERNAL); - CHECK(absl::StrContains(status.error_message(), "exception: mdbx")); + CHECK(absl::StrContains(status.error_message(), "exception: MDBX_ENODATA")); CHECK(responses.size() == 2); CHECK(responses[0].tx_id() != 0); } @@ -1882,7 +1877,7 @@ TEST_CASE("KvServer E2E: Tx cursor invalid operations", "[silkworm][node][rpc]") const auto status = kv_client.tx(requests, responses); CHECK(!status.ok()); CHECK(status.error_code() == ::grpc::StatusCode::INTERNAL); - CHECK(absl::StrContains(status.error_message(), "exception: mdbx")); + CHECK(absl::StrContains(status.error_message(), "exception: MDBX_ENODATA")); CHECK(responses.size() == 2); CHECK(responses[0].tx_id() != 0); } diff --git a/silkworm/db/mdbx_test.cpp b/silkworm/db/mdbx_test.cpp index a6d0466308..ddd33dc454 100644 --- a/silkworm/db/mdbx_test.cpp +++ b/silkworm/db/mdbx_test.cpp @@ -404,9 +404,8 @@ TEST_CASE("Cursor walk") { REQUIRE(table_cursor.size() == kGeneticCode.size()); REQUIRE(table_cursor.empty() == false); - // Rebind cursor so its position is undefined - table_cursor.bind(txn, {kTableName}); - REQUIRE(table_cursor.eof() == true); + // Reset cursor so its position is at the begining + table_cursor.to_first(); // read entire table forward cursor_for_each(table_cursor, save_all_data_map); @@ -587,16 +586,17 @@ TEST_CASE("Cursor walk") { //! Compute the maximum free space in an empty MDBX database page //! \warning this may change in future versions of MDBX //! \details see `page_space` function in MDBX core.c -static size_t page_space(const mdbx::env& env) { +static size_t page_space(const mdbx::txn& txn) { constexpr size_t kPageHeaderSize{20}; // size of MDBX page header (PAGEHDRSZ) - const size_t db_page_size{env.get_pagesize()}; + const auto stats = txn.env().get_stat(txn); + const size_t db_page_size{stats.ms_psize}; return db_page_size - kPageHeaderSize; } static size_t max_multivalue_size_for_leaf_page(const mdbx::txn& txn) { constexpr size_t kDupSortNodes{2}; constexpr size_t kNodeHeaderSize{8}; // size of MDBX node header (NODESIZE) - return page_space(txn.env()) / kDupSortNodes - 2 * kNodeHeaderSize; + return page_space(txn) / kDupSortNodes - 2 * kNodeHeaderSize; } TEST_CASE("OF pages") { @@ -849,10 +849,10 @@ TEST_CASE("Multi-value erase+upsert w/ same value increases free pages") { // We need to split max multi-value data size between key and value constexpr size_t kKeySize{20}; // just to copycat account address size - const size_t max_non_initial_value_size{[&env]() { - auto ro_txn{env.start_read()}; - return max_multivalue_size_for_leaf_page(ro_txn); - }()}; + auto ro_txn{env.start_read()}; + auto stat = env.get_stat(ro_txn); + auto max_non_initial_value_size = max_multivalue_size_for_leaf_page(ro_txn); + ro_txn.abort(); const size_t max_first_value_size{max_non_initial_value_size - kKeySize}; // we need to take key size into account once const Bytes key(kKeySize, '\0'); diff --git a/silkworm/db/memory_mutation_cursor_test.cpp b/silkworm/db/memory_mutation_cursor_test.cpp index 4902de379f..a5572e4c23 100644 --- a/silkworm/db/memory_mutation_cursor_test.cpp +++ b/silkworm/db/memory_mutation_cursor_test.cpp @@ -432,24 +432,12 @@ TEST_CASE("MemoryMutationCursor: to_current_next_multi", "[silkworm][node][db][m SECTION(tag + ": to_current_next_multi on existent table w/ positioning: OK") { MemoryMutationCursor mutation_cursor1{test->mutation, table::kCode}; REQUIRE(mutation_cursor1.to_first()); - const auto result1 = mutation_cursor1.to_current_next_multi(); - CHECK(result1.done); - CHECK(result1.key == "BB"); - CHECK(result1.value == "11"); + const auto result1 = mutation_cursor1.to_current_next_multi(/*throw_notfound=*/false); + CHECK(!result1.done); MemoryMutationCursor mutation_cursor2{test->mutation, table::kCode}; REQUIRE(mutation_cursor2.to_first()); - const auto result2 = mutation_cursor2.to_current_next_multi(/*throw_notfound=*/true); - CHECK(result2.done); - CHECK(result2.key == "BB"); - CHECK(result2.value == "11"); - - MemoryMutationCursor mutation_cursor3{test->mutation, table::kCode}; - REQUIRE(mutation_cursor3.to_first()); - const auto result3 = mutation_cursor3.to_current_next_multi(/*throw_notfound=*/false); - CHECK(result3.done); - CHECK(result3.key == "BB"); - CHECK(result3.value == "11"); + CHECK_THROWS_AS( mutation_cursor2.to_current_next_multi(/*throw_notfound=*/true), mdbx::not_found); MemoryMutationCursor mutation_cursor4{test->mutation, table::kAccountChangeSet}; REQUIRE(mutation_cursor4.to_first()); @@ -477,14 +465,8 @@ TEST_CASE("MemoryMutationCursor: to_current_next_multi", "[silkworm][node][db][m MemoryMutationCursor mutation_cursor1{test->mutation, table::kCode}; REQUIRE(mutation_cursor1.to_first(/*throw_notfound=*/false)); const auto result1 = mutation_cursor1.to_current_next_multi(/*throw_notfound=*/false); - CHECK(result1.done); - CHECK(result1.key == "BB"); - CHECK(result1.value == "11"); - const auto result2 = mutation_cursor1.to_current_next_multi(/*throw_notfound=*/false); - CHECK(!result2.done); - REQUIRE(mutation_cursor1.to_last(/*throw_notfound=*/false)); - const auto result3 = mutation_cursor1.to_current_next_multi(/*throw_notfound=*/false); - CHECK(!result3.done); + CHECK(!result1.done); + CHECK_THROWS_AS(mutation_cursor1.to_current_next_multi(/*throw_notfound=*/true), mdbx::not_found); MemoryMutationCursor mutation_cursor2{test->mutation, table::kAccountChangeSet}; REQUIRE(mutation_cursor2.to_first(/*throw_notfound=*/false)); @@ -1198,19 +1180,19 @@ TEST_CASE("MemoryMutationCursor: SeekBothRange db->mem->db->mem", "[silkworm][no TEST_CASE("MemoryMutationCursor: Delete db->mem->db->mem", "[silkworm][node][db][memory_mutation]") { MemoryMutationCursorTest test; - auto rw_db_cursor = test.main_txn.rw_cursor_dup_sort(table::kHashedAccounts); + auto rw_db_cursor = test.main_txn.rw_cursor(table::kHashedAccounts); rw_db_cursor->upsert(mdbx::slice{"key1"}, mdbx::slice{"value1.1"}); rw_db_cursor->upsert(mdbx::slice{"key3"}, mdbx::slice{"value3.3"}); test.main_txn.commit_and_renew(); - auto rw_mem_cursor = test.mutation.rw_cursor_dup_sort(table::kHashedAccounts); + auto rw_mem_cursor = test.mutation.rw_cursor(table::kHashedAccounts); rw_mem_cursor->upsert("key1", "value1.3"); rw_mem_cursor->upsert("key2", "value2.1"); rw_mem_cursor->upsert("key3", "value3.1"); test.mutation.commit_and_renew(); - auto db_cursor = test.main_txn.rw_cursor_dup_sort(table::kHashedAccounts); - auto mem_cursor = test.mutation.rw_cursor_dup_sort(table::kHashedAccounts); + auto db_cursor = test.main_txn.rw_cursor(table::kHashedAccounts); + auto mem_cursor = test.mutation.rw_cursor(table::kHashedAccounts); mem_cursor->erase(mdbx::slice{"key1"}); mem_cursor->erase(mdbx::slice{"key3"}); From 8062d01010f76cdac58ea498edc75ce9d7756491 Mon Sep 17 00:00:00 2001 From: GitHub Date: Wed, 23 Apr 2025 14:41:47 +0000 Subject: [PATCH 05/10] make fmt --- silkworm/db/memory_mutation_cursor_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/silkworm/db/memory_mutation_cursor_test.cpp b/silkworm/db/memory_mutation_cursor_test.cpp index a5572e4c23..772f63ef45 100644 --- a/silkworm/db/memory_mutation_cursor_test.cpp +++ b/silkworm/db/memory_mutation_cursor_test.cpp @@ -437,7 +437,7 @@ TEST_CASE("MemoryMutationCursor: to_current_next_multi", "[silkworm][node][db][m MemoryMutationCursor mutation_cursor2{test->mutation, table::kCode}; REQUIRE(mutation_cursor2.to_first()); - CHECK_THROWS_AS( mutation_cursor2.to_current_next_multi(/*throw_notfound=*/true), mdbx::not_found); + CHECK_THROWS_AS(mutation_cursor2.to_current_next_multi(/*throw_notfound=*/true), mdbx::not_found); MemoryMutationCursor mutation_cursor4{test->mutation, table::kAccountChangeSet}; REQUIRE(mutation_cursor4.to_first()); From 371a90a41d4b00f7590977ec24b4ee8d43d264ad Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Wed, 23 Apr 2025 16:52:40 +0200 Subject: [PATCH 06/10] Fix tests for libmdbx 0.13 --- silkworm/db/mdbx_test.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/silkworm/db/mdbx_test.cpp b/silkworm/db/mdbx_test.cpp index ddd33dc454..eda2b57563 100644 --- a/silkworm/db/mdbx_test.cpp +++ b/silkworm/db/mdbx_test.cpp @@ -850,7 +850,6 @@ TEST_CASE("Multi-value erase+upsert w/ same value increases free pages") { // We need to split max multi-value data size between key and value constexpr size_t kKeySize{20}; // just to copycat account address size auto ro_txn{env.start_read()}; - auto stat = env.get_stat(ro_txn); auto max_non_initial_value_size = max_multivalue_size_for_leaf_page(ro_txn); ro_txn.abort(); const size_t max_first_value_size{max_non_initial_value_size - kKeySize}; // we need to take key size into account once From 354dba9a9833eaa8c2f6cdd44752c4b9199ff70a Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Thu, 24 Apr 2025 10:16:08 +0200 Subject: [PATCH 07/10] Update CLang to 18 --- .circleci/config.yml | 4 ++-- .github/actions/fuzzer-common-steps/action.yml | 4 ++-- ...{linux_x64_clang_16_debug => linux_x64_clang_18_debug} | 0 ...ux_x64_clang_16_release => linux_x64_clang_18_release} | 2 +- docs/fuzzer.md | 2 +- silkworm/db/datastore/kvdb/mdbx.cpp | 4 ++-- tests/docker/Dockerfile | 8 ++++---- 7 files changed, 12 insertions(+), 12 deletions(-) rename cmake/profiles/{linux_x64_clang_16_debug => linux_x64_clang_18_debug} (100%) rename cmake/profiles/{linux_x64_clang_16_release => linux_x64_clang_18_release} (84%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 86eb871fcc..68051b3256 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -95,7 +95,7 @@ commands: command: | if [[ "<>" == "clang" ]] then - conan_profile=linux_x64_clang_16_release + conan_profile=linux_x64_clang_18_release else conan_profile=linux_x64_gcc_11_release fi @@ -132,7 +132,7 @@ commands: - run: name: "CMake Fuzzer" working_directory: ~/build - command: cmake ../project -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_16_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON + command: cmake ../project -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_18_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON - run: name: "Build Fuzzer" command: | diff --git a/.github/actions/fuzzer-common-steps/action.yml b/.github/actions/fuzzer-common-steps/action.yml index d524d19e52..d8aa868b17 100644 --- a/.github/actions/fuzzer-common-steps/action.yml +++ b/.github/actions/fuzzer-common-steps/action.yml @@ -11,7 +11,7 @@ runs: steps: - name: Install Compiler shell: bash - run: cmake/setup/compiler_install.sh clang 16 + run: cmake/setup/compiler_install.sh clang 18 - name: Clean Build Directory shell: bash @@ -28,7 +28,7 @@ runs: - name: Configure CMake shell: bash working-directory: ${{runner.workspace}}/silkworm/build - run: CC=clang-16 CXX=clang++-16 cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_16_release -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON -DSILKWORM_FUZZER_SANITIZERS=${{inputs.fuzzer_sanitizers}} + run: CC=clang-16 CXX=clang++-16 cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_18_release -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON -DSILKWORM_FUZZER_SANITIZERS=${{inputs.fuzzer_sanitizers}} - name: Build Silkworm Fuzzer Test shell: bash diff --git a/cmake/profiles/linux_x64_clang_16_debug b/cmake/profiles/linux_x64_clang_18_debug similarity index 100% rename from cmake/profiles/linux_x64_clang_16_debug rename to cmake/profiles/linux_x64_clang_18_debug diff --git a/cmake/profiles/linux_x64_clang_16_release b/cmake/profiles/linux_x64_clang_18_release similarity index 84% rename from cmake/profiles/linux_x64_clang_16_release rename to cmake/profiles/linux_x64_clang_18_release index 3164500a5a..4db13e30f0 100644 --- a/cmake/profiles/linux_x64_clang_16_release +++ b/cmake/profiles/linux_x64_clang_18_release @@ -2,7 +2,7 @@ os=Linux arch=x86_64 compiler=clang -compiler.version=16 +compiler.version=18 compiler.libcxx=libc++ compiler.cppstd=17 build_type=Release diff --git a/docs/fuzzer.md b/docs/fuzzer.md index 8a8553ce65..e5f284ba3f 100644 --- a/docs/fuzzer.md +++ b/docs/fuzzer.md @@ -6,7 +6,7 @@ To build the fuzzer use the following: ```bash mkdir build cd build -cmake ../project -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_16_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON +cmake ../project -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_18_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON cmake --build --target rpcdaemon_fuzzer_test cmake –-build --target rpcdaemon_fuzzer_diagnostics ``` diff --git a/silkworm/db/datastore/kvdb/mdbx.cpp b/silkworm/db/datastore/kvdb/mdbx.cpp index a18c6c0f68..d3440e69a3 100644 --- a/silkworm/db/datastore/kvdb/mdbx.cpp +++ b/silkworm/db/datastore/kvdb/mdbx.cpp @@ -21,7 +21,7 @@ namespace detail { } std::string slice_as_hex(const Slice& data) { - return ::mdbx::to_hex(data).as_string(); + return std::string(::mdbx::to_hex(data).as_string()); } log::Args log_args_for_commit_latency(const MDBX_commit_latency& commit_latency) { @@ -577,7 +577,7 @@ std::vector list_maps(::mdbx::txn& tx, bool throw_notfound) { ::mdbx::map_handle main_map{1}; auto main_cursor{tx.open_cursor(main_map)}; for (auto it{main_cursor.to_first(throw_notfound)}; it.done; it = main_cursor.to_next(throw_notfound)) { - map_names.push_back(it.key.as_string()); + map_names.push_back(std::string(it.key.as_string())); } return map_names; } diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index 3e6284645c..3a5b6028c8 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -18,11 +18,11 @@ RUN git submodule sync RUN git submodule update --init --recursive # 3 Install compiler -RUN cmake/setup/compiler_install.sh clang 16 +RUN cmake/setup/compiler_install.sh clang 18 # 4 Build all targets WORKDIR /app/build -RUN cmake ../project -DCMAKE_BUILD_TYPE=Debug -DCONAN_PROFILE=linux_x64_clang_16_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake +RUN cmake ../project -DCMAKE_BUILD_TYPE=Debug -DCONAN_PROFILE=linux_x64_clang_18_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake WORKDIR /app/project RUN cmake --build /app/build -j4 @@ -48,7 +48,7 @@ RUN git pull origin ${BRANCH} # Rebuild all targets RUN echo "Rebuilding" WORKDIR /app/build -RUN sudo cmake ../project -DCMAKE_BUILD_TYPE=Debug -DCONAN_PROFILE=linux_x64_clang_16_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake +RUN sudo cmake ../project -DCMAKE_BUILD_TYPE=Debug -DCONAN_PROFILE=linux_x64_clang_18_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake WORKDIR /app/project RUN sudo cmake --build /app/build -j4 @@ -64,7 +64,7 @@ RUN sudo cmake --build /app/build -j4 # RUN for file in ~/artifacts/*; do sed -i 's/^>> //' "$file"; done # # Rebuild fuzzer # WORKDIR /app/build -# RUN sudo cmake ../project -DCMAKE_BUILD_TYPE=Debug -DCONAN_PROFILE=linux_x64_clang_16_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON +# RUN sudo cmake ../project -DCMAKE_BUILD_TYPE=Debug -DCONAN_PROFILE=linux_x64_clang_18_release -DCMAKE_TOOLCHAIN_FILE=../project/cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON # WORKDIR /app/project # RUN sudo cmake --build /app/build -j4 --target rpcdaemon_fuzzer_test # # Run fuzzer From 96c342108de92fabf181fac09f91521a409a1dbd Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Thu, 24 Apr 2025 11:11:17 +0200 Subject: [PATCH 08/10] Fix GCC compilation errors --- silkworm/db/datastore/kvdb/mdbx.cpp | 8 ++++++++ silkworm/db/datastore/kvdb/mdbx.hpp | 2 ++ silkworm/db/kv/api/local_cursor.cpp | 21 +++++++++++---------- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/silkworm/db/datastore/kvdb/mdbx.cpp b/silkworm/db/datastore/kvdb/mdbx.cpp index d3440e69a3..598921604d 100644 --- a/silkworm/db/datastore/kvdb/mdbx.cpp +++ b/silkworm/db/datastore/kvdb/mdbx.cpp @@ -24,6 +24,14 @@ namespace detail { return std::string(::mdbx::to_hex(data).as_string()); } + std::string slice_as_string(const Slice& data) { + return std::string(data.as_string()); + } + + silkworm::Bytes slice_as_bytes(const Slice& data) { + return silkworm::Bytes{reinterpret_cast(data.data()), data.size()}; + } + log::Args log_args_for_commit_latency(const MDBX_commit_latency& commit_latency) { return { "preparation", diff --git a/silkworm/db/datastore/kvdb/mdbx.hpp b/silkworm/db/datastore/kvdb/mdbx.hpp index af6264562c..14b343cfb6 100644 --- a/silkworm/db/datastore/kvdb/mdbx.hpp +++ b/silkworm/db/datastore/kvdb/mdbx.hpp @@ -60,6 +60,8 @@ namespace detail { std::string dump_mdbx_result(const CursorResult& result); std::string slice_as_hex(const Slice& data); + std::string slice_as_string(const Slice& data); + silkworm::Bytes slice_as_bytes(const Slice& data); } // namespace detail class ROTxn; diff --git a/silkworm/db/kv/api/local_cursor.cpp b/silkworm/db/kv/api/local_cursor.cpp index 5e07157153..769f9c7b8f 100644 --- a/silkworm/db/kv/api/local_cursor.cpp +++ b/silkworm/db/kv/api/local_cursor.cpp @@ -12,6 +12,7 @@ namespace silkworm::db::kv::api { using namespace silkworm::datastore::kvdb; using datastore::kvdb::detail::slice_as_hex; +using datastore::kvdb::detail::slice_as_bytes; Task LocalCursor::open_cursor(std::string_view table_name, bool is_dup_sorted) { const auto start_time = clock_time::now(); @@ -33,7 +34,7 @@ Task LocalCursor::seek(ByteView key) { if (result) { SILK_DEBUG << "LocalCursor::seek found: key: " << key << " value: " << slice_as_hex(result.value); - co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())}; + co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)}; } else { SILK_DEBUG << "LocalCursor::seek not found key: " << key; co_return KeyValue{}; @@ -49,7 +50,7 @@ Task LocalCursor::seek_exact(ByteView key) { SILK_DEBUG << "LocalCursor::seek_exact result: " << detail::dump_mdbx_result(result); if (result) { SILK_DEBUG << "LocalCursor::seek_exact found: key: " << key << " value: " << slice_as_hex(result.value); - co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())}; + co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)}; } SILK_ERROR << "LocalCursor::seek_exact !result key: " << key; // TODO(canepat) handle properly? } @@ -66,7 +67,7 @@ Task LocalCursor::first() { } SILK_DEBUG << "LocalCursor::first: key: " << slice_as_hex(result.key) << " value: " << slice_as_hex(result.value); - co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())}; + co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)}; } Task LocalCursor::last() { @@ -79,7 +80,7 @@ Task LocalCursor::last() { } SILK_DEBUG << "LocalCursor::last: key: " << slice_as_hex(result.key) << " value: " << slice_as_hex(result.value); - co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())}; + co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)}; } Task LocalCursor::next() { @@ -91,7 +92,7 @@ Task LocalCursor::next() { if (result) { SILK_DEBUG << "LocalCursor::next: " << " key: " << string_view_to_byte_view(result.key.as_string()) << " value: " << string_view_to_byte_view(result.value.as_string()); - co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())}; + co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)}; } else { SILK_ERROR << "LocalCursor::next !result"; // TODO(canepat) handle properly? } @@ -107,7 +108,7 @@ Task LocalCursor::previous() { if (result) { SILK_DEBUG << "LocalCursor::previous: " << " key: " << string_view_to_byte_view(result.key.as_string()) << " value: " << string_view_to_byte_view(result.value.as_string()); - co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())}; + co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)}; } else { SILK_ERROR << "LocalCursor::previous !result"; // TODO(canepat) handle properly? } @@ -123,7 +124,7 @@ Task LocalCursor::next_dup() { if (result) { SILK_DEBUG << "LocalCursor::next_dup: " << " key: " << string_view_to_byte_view(result.key.as_string()) << " value: " << string_view_to_byte_view(result.value.as_string()); - co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())}; + co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)}; } else { SILK_ERROR << "LocalCursor::next_dup !result"; // TODO(canepat) handle properly? } @@ -138,9 +139,9 @@ Task LocalCursor::seek_both(ByteView key, ByteView value) { if (result) { SILK_DEBUG << "LocalCursor::seek_both key: " << string_view_to_byte_view(result.key.as_string()) << " value: " << string_view_to_byte_view(result.value.as_string()); - co_return string_to_bytes(result.value.as_string()); + co_return slice_as_bytes(result.value); } - co_return string_to_bytes(""); + co_return Bytes{}; } Task LocalCursor::seek_both_exact(ByteView key, ByteView value) { @@ -152,7 +153,7 @@ Task LocalCursor::seek_both_exact(ByteView key, ByteView value) { if (result) { SILK_DEBUG << "LocalCursor::seek_both_exact: " << " key: " << string_view_to_byte_view(result.key.as_string()) << " value: " << string_view_to_byte_view(result.value.as_string()); - co_return KeyValue{string_to_bytes(result.key.as_string()), string_to_bytes(result.value.as_string())}; + co_return KeyValue{slice_as_bytes(result.key), slice_as_bytes(result.value)}; } else { SILK_ERROR << "LocalCursor::seek_both_exact !found key: " << key << " subkey:" << value; } From 21f05354399c613c47759665bbb4342b737c4f75 Mon Sep 17 00:00:00 2001 From: GitHub Date: Thu, 24 Apr 2025 09:11:58 +0000 Subject: [PATCH 09/10] make fmt --- silkworm/db/kv/api/local_cursor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/silkworm/db/kv/api/local_cursor.cpp b/silkworm/db/kv/api/local_cursor.cpp index 769f9c7b8f..9e3feea460 100644 --- a/silkworm/db/kv/api/local_cursor.cpp +++ b/silkworm/db/kv/api/local_cursor.cpp @@ -11,8 +11,8 @@ namespace silkworm::db::kv::api { using namespace silkworm::datastore::kvdb; -using datastore::kvdb::detail::slice_as_hex; using datastore::kvdb::detail::slice_as_bytes; +using datastore::kvdb::detail::slice_as_hex; Task LocalCursor::open_cursor(std::string_view table_name, bool is_dup_sorted) { const auto start_time = clock_time::now(); From 449acec2395588e778650b72255be03134df5e81 Mon Sep 17 00:00:00 2001 From: Jacek Glen Date: Thu, 24 Apr 2025 11:26:23 +0200 Subject: [PATCH 10/10] Fix GCC compilation errors --- silkworm/db/datastore/kvdb/mdbx.cpp | 2 +- silkworm/db/kv/grpc/server/kv_calls.cpp | 3 ++- silkworm/db/prune_mode.cpp | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/silkworm/db/datastore/kvdb/mdbx.cpp b/silkworm/db/datastore/kvdb/mdbx.cpp index 598921604d..fdcda05348 100644 --- a/silkworm/db/datastore/kvdb/mdbx.cpp +++ b/silkworm/db/datastore/kvdb/mdbx.cpp @@ -585,7 +585,7 @@ std::vector list_maps(::mdbx::txn& tx, bool throw_notfound) { ::mdbx::map_handle main_map{1}; auto main_cursor{tx.open_cursor(main_map)}; for (auto it{main_cursor.to_first(throw_notfound)}; it.done; it = main_cursor.to_next(throw_notfound)) { - map_names.push_back(std::string(it.key.as_string())); + map_names.push_back(detail::slice_as_string(it.key)); } return map_names; } diff --git a/silkworm/db/kv/grpc/server/kv_calls.cpp b/silkworm/db/kv/grpc/server/kv_calls.cpp index 52c57bc4d0..461d1d312c 100644 --- a/silkworm/db/kv/grpc/server/kv_calls.cpp +++ b/silkworm/db/kv/grpc/server/kv_calls.cpp @@ -24,6 +24,7 @@ using boost::asio::as_tuple; using namespace boost::asio::experimental::awaitable_operators; using boost::asio::steady_timer; using boost::asio::use_awaitable; +using datastore::kvdb::detail::slice_as_string; api::Version higher_version_ignoring_patch(api::Version lhs, api::Version rhs) { uint32_t lhs_major = std::get<0>(lhs); @@ -350,7 +351,7 @@ bool TxCall::save_cursors(std::vector& positions) { } mdbx::slice key = result.key; mdbx::slice value = result.value; - positions.emplace_back(CursorPosition{key.as_string(), value.as_string()}); + positions.emplace_back(CursorPosition{slice_as_string(key), slice_as_string(value)}); } } diff --git a/silkworm/db/prune_mode.cpp b/silkworm/db/prune_mode.cpp index a22a175c8d..779b7734b4 100644 --- a/silkworm/db/prune_mode.cpp +++ b/silkworm/db/prune_mode.cpp @@ -36,7 +36,7 @@ static BlockAmount read_block_amount_for_key(mdbx::cursor& source, std::string_v value_type = BlockAmount::Type::kBefore; } else { // Something bad has been written - throw std::runtime_error("Invalid prune type stored in database : " + data2.value.as_string()); + throw std::runtime_error("Invalid prune type stored in database : " + std::string(data2.value.as_string())); } } return BlockAmount(value_type, value);