Skip to content

Commit 0f155d8

Browse files
committed
SDSTOR-21465: scrubber phase 1
1 parent d9009cf commit 0f155d8

31 files changed

Lines changed: 5277 additions & 102 deletions

CHANGELOG.md

Lines changed: 0 additions & 13 deletions
This file was deleted.

conanfile.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
class HomeObjectConan(ConanFile):
1212
name = "homeobject"
13-
version = "4.1.9"
13+
version = "4.2.0"
1414

1515
homepage = "https://github.com/eBay/HomeObject"
1616
description = "Blob Store built on HomeStore"

src/include/homeobject/common.hpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414

1515
SISL_LOGGING_DECL(homeobject);
1616

17-
#define HOMEOBJECT_LOG_MODS homeobject, blobmgr, shardmgr, gcmgr
17+
#define HOMEOBJECT_LOG_MODS homeobject, blobmgr, shardmgr, gcmgr, scrubmgr
1818

1919
#ifndef Ki
2020
constexpr uint64_t Ki = 1024ul;

src/lib/homeobject_impl.hpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,8 @@ class HomeObjectImpl : public HomeObject,
8888
public std::enable_shared_from_this< HomeObjectImpl > {
8989

9090
/// Implementation defines these
91-
virtual ShardManager::AsyncResult< ShardInfo > _create_shard(pg_id_t, uint64_t size_bytes, std::string meta, trace_id_t tid) = 0;
91+
virtual ShardManager::AsyncResult< ShardInfo > _create_shard(pg_id_t, uint64_t size_bytes, std::string meta,
92+
trace_id_t tid) = 0;
9293
virtual ShardManager::AsyncResult< ShardInfo > _seal_shard(ShardInfo const&, trace_id_t tid) = 0;
9394

9495
virtual BlobManager::AsyncResult< blob_id_t > _put_blob(ShardInfo const&, Blob&&, trace_id_t tid) = 0;
@@ -189,7 +190,8 @@ class HomeObjectImpl : public HomeObject,
189190

190191
/// ShardManager
191192
ShardManager::AsyncResult< ShardInfo > get_shard(shard_id_t id, trace_id_t tid) const final;
192-
ShardManager::AsyncResult< ShardInfo > create_shard(pg_id_t pg_owner, uint64_t size_bytes, std::string meta, trace_id_t tid) final;
193+
ShardManager::AsyncResult< ShardInfo > create_shard(pg_id_t pg_owner, uint64_t size_bytes, std::string meta,
194+
trace_id_t tid) final;
193195
ShardManager::AsyncResult< InfoList > list_shards(pg_id_t pg, trace_id_t tid) const final;
194196
ShardManager::AsyncResult< ShardInfo > seal_shard(shard_id_t id, trace_id_t tid) final;
195197
uint64_t get_current_timestamp();

src/lib/homestore_backend/CMakeLists.txt

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ target_sources("${PROJECT_NAME}_homestore" PRIVATE
3030
hs_cp_callbacks.cpp
3131
hs_http_manager.cpp
3232
gc_manager.cpp
33+
scrub_manager.cpp
34+
MPMCPriorityQueue.hpp
3335
$<TARGET_OBJECTS:${PROJECT_NAME}_core>
3436
)
3537
target_link_libraries("${PROJECT_NAME}_homestore" PUBLIC
@@ -42,10 +44,14 @@ settings_gen_cpp(
4244
${FLATBUFFERS_FLATC_EXECUTABLE}
4345
${CMAKE_CURRENT_BINARY_DIR}/generated/
4446
"${PROJECT_NAME}_homestore"
45-
hs_backend_config.fbs
46-
resync_pg_data.fbs
47-
resync_shard_data.fbs
48-
resync_blob_data.fbs
47+
hs_homeobject_fbs/hs_backend_config.fbs
48+
hs_homeobject_fbs/resync_pg_data.fbs
49+
hs_homeobject_fbs/resync_shard_data.fbs
50+
hs_homeobject_fbs/resync_blob_data.fbs
51+
52+
hs_homeobject_fbs/scrub_common.fbs
53+
hs_homeobject_fbs/scrub_req.fbs
54+
hs_homeobject_fbs/scrub_result.fbs
4955
)
5056

5157
# Unit test objects
@@ -165,3 +171,12 @@ add_test(NAME HomestoreTestGC COMMAND homestore_test_gc -csv error --executor im
165171
--override_config hs_backend_config.gc_enable_read_verify=true
166172
--override_config hs_backend_config.gc_garbage_rate_threshold=0
167173
--override_config hs_backend_config.gc_scan_interval_sec=5)
174+
175+
add_executable(homestore_test_scrubber)
176+
target_sources(homestore_test_scrubber PRIVATE $<TARGET_OBJECTS:homestore_tests_scrubber>)
177+
target_link_libraries(homestore_test_scrubber PUBLIC homeobject_homestore ${COMMON_TEST_DEPS})
178+
add_test(NAME HomestoreTestScrubber COMMAND homestore_test_scrubber -csv error --executor immediate --config_path ./
179+
--override_config hs_backend_config.enable_scrubber=true
180+
--override_config nuraft_mesg_config.mesg_factory_config.data_request_deadline_secs:10)
181+
182+
Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
#pragma once
2+
3+
#include <condition_variable>
4+
#include <concepts>
5+
#include <cstddef>
6+
#include <functional>
7+
#include <mutex>
8+
#include <optional>
9+
#include <queue>
10+
#include <utility>
11+
#include <vector>
12+
13+
namespace homeobject {
14+
15+
/**
16+
* @brief Multi-Producer Multi-Consumer Priority Queue (C++20)
17+
*
18+
* Thread-safe priority queue that supports:
19+
* - Concurrent push operations from multiple producers
20+
* - Concurrent pop operations from multiple consumers
21+
* - Blocking pop when queue is empty
22+
* - Graceful shutdown via close() method
23+
*
24+
* @tparam T Element type (must be comparable)
25+
* @tparam Compare Comparison function (default: std::less for max-heap)
26+
*/
27+
template < typename T, typename Compare = std::less< T > >
28+
requires std::movable< T > && std::predicate< Compare, T, T >
29+
class MPMCPriorityQueue {
30+
public:
31+
using value_type = T;
32+
using size_type = std::size_t;
33+
using comparator_type = Compare;
34+
35+
/**
36+
* @brief Status codes returned by pop operations
37+
*/
38+
enum class Status : uint8_t {
39+
Ok, ///< Successfully popped an element
40+
Closed ///< Queue is closed, no more elements available
41+
};
42+
43+
/**
44+
* @brief Result of a pop operation
45+
*/
46+
struct PopResult {
47+
Status status;
48+
std::optional< T > value; ///< Has value only if status == Ok
49+
50+
// Convenience methods
51+
[[nodiscard]] constexpr bool is_ok() const noexcept { return status == Status::Ok; }
52+
[[nodiscard]] constexpr bool is_closed() const noexcept { return status == Status::Closed; }
53+
};
54+
55+
/**
56+
* @brief Construct an empty priority queue
57+
*/
58+
constexpr MPMCPriorityQueue() noexcept(std::is_nothrow_default_constructible_v< Compare >) = default;
59+
60+
/**
61+
* @brief Destructor - automatically closes the queue
62+
*/
63+
~MPMCPriorityQueue() { close(); }
64+
65+
// Disable copy and move to prevent issues with condition variables
66+
MPMCPriorityQueue(const MPMCPriorityQueue&) = delete;
67+
MPMCPriorityQueue& operator=(const MPMCPriorityQueue&) = delete;
68+
MPMCPriorityQueue(MPMCPriorityQueue&&) = delete;
69+
MPMCPriorityQueue& operator=(MPMCPriorityQueue&&) = delete;
70+
71+
/**
72+
* @brief Thread-safe push operation (copy)
73+
*
74+
* @param value Element to insert
75+
* @return true if pushed successfully, false if queue is closed
76+
*/
77+
bool push(const T& value)
78+
requires std::copy_constructible< T >
79+
{
80+
{
81+
std::scoped_lock lock(mutex_);
82+
if (closed_) [[unlikely]] {
83+
return false; // Queue is closed, cannot push
84+
}
85+
pq_.push(value);
86+
}
87+
cv_.notify_one(); // Wake one waiting consumer
88+
return true;
89+
}
90+
91+
/**
92+
* @brief Thread-safe push operation (move)
93+
*
94+
* @param value Element to insert (will be moved)
95+
* @return true if pushed successfully, false if queue is closed
96+
*/
97+
bool push(T&& value) {
98+
{
99+
std::scoped_lock lock(mutex_);
100+
if (closed_) [[unlikely]] { return false; }
101+
pq_.push(std::move(value));
102+
}
103+
cv_.notify_one();
104+
return true;
105+
}
106+
107+
/**
108+
* @brief Thread-safe pop operation
109+
*
110+
* Blocks if queue is empty and not closed.
111+
* Returns immediately if queue is closed.
112+
*
113+
* @return PopResult containing status and optional value
114+
* @note Thread-safe for multiple concurrent consumers
115+
*/
116+
[[nodiscard]] PopResult pop() {
117+
std::unique_lock lock(mutex_);
118+
119+
// Wait until queue has elements or is closed
120+
cv_.wait(lock, [this] { return closed_ || !pq_.empty(); });
121+
122+
// Try to pop an element
123+
if (!pq_.empty()) {
124+
T top = std::move(const_cast< T& >(pq_.top()));
125+
pq_.pop();
126+
return PopResult{.status = Status::Ok, .value = std::move(top)};
127+
}
128+
129+
// Queue is empty and closed
130+
return PopResult{.status = Status::Closed, .value = std::nullopt};
131+
}
132+
133+
/**
134+
* @brief Close the queue
135+
*
136+
* After calling close():
137+
* - All blocked pop() calls will wake up
138+
* - Existing elements can still be popped
139+
* - New push() calls will be ignored
140+
* - pop() returns Status::Closed when queue becomes empty
141+
*
142+
* @note Thread-safe and idempotent
143+
*/
144+
void close() noexcept {
145+
{
146+
std::scoped_lock lock(mutex_);
147+
closed_ = true;
148+
}
149+
cv_.notify_all(); // Wake all waiting consumers
150+
}
151+
152+
/**
153+
* @brief Get current number of elements
154+
*
155+
* @return Number of elements in the queue
156+
* @note Thread-safe
157+
*/
158+
[[nodiscard]] size_type size() const {
159+
std::scoped_lock lock(mutex_);
160+
return pq_.size();
161+
}
162+
163+
/**
164+
* @brief Check if queue is empty
165+
*
166+
* @return true if queue has no elements
167+
* @note Thread-safe
168+
*/
169+
[[nodiscard]] bool empty() const {
170+
std::scoped_lock lock(mutex_);
171+
return pq_.empty();
172+
}
173+
174+
/**
175+
* @brief Check if queue is closed
176+
*
177+
* @return true if close() has been called
178+
* @note Thread-safe
179+
*/
180+
[[nodiscard]] bool is_closed() const {
181+
std::scoped_lock lock(mutex_);
182+
return closed_;
183+
}
184+
185+
private:
186+
mutable std::mutex mutex_;
187+
std::condition_variable cv_;
188+
bool closed_{false};
189+
std::priority_queue< T, std::vector< T >, Compare > pq_;
190+
};
191+
192+
} // namespace homeobject

src/lib/homestore_backend/gc_manager.cpp

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -25,14 +25,14 @@ SISL_LOGGING_DECL(gcmgr)
2525
GCManager::GCManager(HSHomeObject* homeobject) :
2626
m_chunk_selector{homeobject->chunk_selector()}, m_hs_home_object{homeobject} {
2727
homestore::meta_service().register_handler(
28-
_gc_actor_meta_name,
28+
gc_actor_meta_name,
2929
[this](homestore::meta_blk* mblk, sisl::byte_view buf, size_t size) {
3030
on_gc_actor_meta_blk_found(std::move(buf), voidptr_cast(mblk));
3131
},
3232
nullptr, true);
3333

3434
homestore::meta_service().register_handler(
35-
_gc_reserved_chunk_meta_name,
35+
gc_reserved_chunk_meta_name,
3636
[this](homestore::meta_blk* mblk, sisl::byte_view buf, size_t size) {
3737
on_reserved_chunk_meta_blk_found(std::move(buf), voidptr_cast(mblk));
3838
},
@@ -44,7 +44,7 @@ GCManager::GCManager(HSHomeObject* homeobject) :
4444
true);
4545

4646
homestore::meta_service().register_handler(
47-
_gc_task_meta_name,
47+
gc_task_meta_name,
4848
[this](homestore::meta_blk* mblk, sisl::byte_view buf, size_t size) {
4949
on_gc_task_meta_blk_found(std::move(buf), voidptr_cast(mblk));
5050
},
@@ -64,7 +64,7 @@ void GCManager::on_gc_task_meta_blk_found(sisl::byte_view const& buf, void* meta
6464
// here, we are under the protection of the lock of metaservice. however, we will also try to update pg and shard
6565
// metablk and then destroy the gc_task_sb, which will also try to acquire the lock of metaservice, as a result, a
6666
// dead lock will happen. so here we will handle all the gc tasks after read all the metablks
67-
m_recovered_gc_tasks.emplace_back(_gc_task_meta_name);
67+
m_recovered_gc_tasks.emplace_back(gc_task_meta_name);
6868
m_recovered_gc_tasks.back().load(buf, meta_cookie);
6969
}
7070

@@ -89,7 +89,7 @@ void GCManager::handle_all_recovered_gc_tasks() {
8989
}
9090

9191
void GCManager::on_gc_actor_meta_blk_found(sisl::byte_view const& buf, void* meta_cookie) {
92-
m_gc_actor_sbs.emplace_back(_gc_actor_meta_name);
92+
m_gc_actor_sbs.emplace_back(gc_actor_meta_name);
9393
auto& gc_actor_sb = m_gc_actor_sbs.back();
9494
gc_actor_sb.load(buf, meta_cookie);
9595
auto pdev_id = gc_actor_sb->pdev_id;
@@ -100,7 +100,7 @@ void GCManager::on_gc_actor_meta_blk_found(sisl::byte_view const& buf, void* met
100100
}
101101

102102
void GCManager::on_reserved_chunk_meta_blk_found(sisl::byte_view const& buf, void* meta_cookie) {
103-
homestore::superblk< gc_reserved_chunk_superblk > reserved_chunk_sb(_gc_reserved_chunk_meta_name);
103+
homestore::superblk< gc_reserved_chunk_superblk > reserved_chunk_sb(gc_reserved_chunk_meta_name);
104104
auto chunk_id = reserved_chunk_sb.load(buf, meta_cookie)->chunk_id;
105105
auto EXVchunk = m_chunk_selector->get_extend_vchunk(chunk_id);
106106
if (EXVchunk == nullptr) {
@@ -976,7 +976,7 @@ bool GCManager::pdev_gc_actor::copy_valid_data(
976976

977977
if (err) {
978978
// we will come here if:
979-
// 1 any blob copy fails, then err is operation_canceled
979+
// 1 any blob copy fails, then err is operation_cancelled
980980
// 2 write footer fails, then err is the error code of write footer
981981
GCLOGE(task_id, pg_id, shard_id,
982982
"Failed to copy some blos or failed to write shard footer for move_to_chunk={}, "
@@ -1271,7 +1271,7 @@ void GCManager::pdev_gc_actor::process_gc_task(chunk_id_t move_from_chunk, uint8
12711271

12721272
// after data copy, we persist the gc task meta blk. now, we can make sure all the valid blobs are successfully
12731273
// copyed and new blob indexes have be written to gc index table before gc task superblk is persisted.
1274-
homestore::superblk< GCManager::gc_task_superblk > gc_task_sb{GCManager::_gc_task_meta_name};
1274+
homestore::superblk< GCManager::gc_task_superblk > gc_task_sb{GCManager::gc_task_meta_name};
12751275
gc_task_sb.create(sizeof(GCManager::gc_task_superblk));
12761276
gc_task_sb->move_from_chunk = move_from_chunk;
12771277
gc_task_sb->move_to_chunk = move_to_chunk;

src/lib/homestore_backend/gc_manager.hpp

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,9 @@ class GCManager {
4646
GCManager& operator=(GCManager&&) = delete;
4747

4848
public:
49-
inline static auto const _gc_actor_meta_name = std::string("GCActor");
50-
inline static auto const _gc_task_meta_name = std::string("GCTask");
51-
inline static auto const _gc_reserved_chunk_meta_name = std::string("GCReservedChunk");
49+
inline static auto const gc_actor_meta_name = std::string("GCActor");
50+
inline static auto const gc_task_meta_name = std::string("GCTask");
51+
inline static auto const gc_reserved_chunk_meta_name = std::string("GCReservedChunk");
5252
inline static atomic_uint64_t _gc_task_id{1}; // 0 is used for crash recovery
5353

5454
#pragma pack(1)
@@ -61,7 +61,7 @@ class GCManager {
6161
uint64_t failed_egc_task_count{0ull};
6262
uint64_t total_reclaimed_blk_count_by_gc{0ull};
6363
uint64_t total_reclaimed_blk_count_by_egc{0ull};
64-
static std::string name() { return _gc_actor_meta_name; }
64+
static std::string name() { return gc_actor_meta_name; }
6565
};
6666

6767
struct gc_task_superblk {
@@ -70,12 +70,12 @@ class GCManager {
7070
chunk_id_t vchunk_id;
7171
pg_id_t pg_id;
7272
uint8_t priority;
73-
static std::string name() { return _gc_task_meta_name; }
73+
static std::string name() { return gc_task_meta_name; }
7474
};
7575

7676
struct gc_reserved_chunk_superblk {
7777
chunk_id_t chunk_id;
78-
static std::string name() { return _gc_reserved_chunk_meta_name; }
78+
static std::string name() { return gc_reserved_chunk_meta_name; }
7979
};
8080
#pragma pack()
8181

0 commit comments

Comments
 (0)