Skip to content

Commit 003dfd5

Browse files
committed
SDSTOR-21465: scrubber phase 1
1 parent e1c23e1 commit 003dfd5

29 files changed

Lines changed: 5265 additions & 78 deletions

CHANGELOG.md

Lines changed: 0 additions & 13 deletions
This file was deleted.

conanfile.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
class HomeObjectConan(ConanFile):
1212
name = "homeobject"
13-
version = "4.1.18"
13+
version = "4.2.0"
1414

1515
homepage = "https://github.com/eBay/HomeObject"
1616
description = "Blob Store built on HomeStore"

src/include/homeobject/common.hpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414

1515
SISL_LOGGING_DECL(homeobject);
1616

17-
#define HOMEOBJECT_LOG_MODS homeobject, blobmgr, shardmgr, gcmgr
17+
#define HOMEOBJECT_LOG_MODS homeobject, blobmgr, shardmgr, gcmgr, scrubmgr
1818

1919
#ifndef Ki
2020
constexpr uint64_t Ki = 1024ul;

src/lib/homeobject_impl.hpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,8 @@ class HomeObjectImpl : public HomeObject,
8888
public std::enable_shared_from_this< HomeObjectImpl > {
8989

9090
/// Implementation defines these
91-
virtual ShardManager::AsyncResult< ShardInfo > _create_shard(pg_id_t, uint64_t size_bytes, std::string meta, trace_id_t tid) = 0;
91+
virtual ShardManager::AsyncResult< ShardInfo > _create_shard(pg_id_t, uint64_t size_bytes, std::string meta,
92+
trace_id_t tid) = 0;
9293
virtual ShardManager::AsyncResult< ShardInfo > _seal_shard(ShardInfo const&, trace_id_t tid) = 0;
9394

9495
virtual BlobManager::AsyncResult< blob_id_t > _put_blob(ShardInfo const&, Blob&&, trace_id_t tid) = 0;
@@ -189,7 +190,8 @@ class HomeObjectImpl : public HomeObject,
189190

190191
/// ShardManager
191192
ShardManager::AsyncResult< ShardInfo > get_shard(shard_id_t id, trace_id_t tid) const final;
192-
ShardManager::AsyncResult< ShardInfo > create_shard(pg_id_t pg_owner, uint64_t size_bytes, std::string meta, trace_id_t tid) final;
193+
ShardManager::AsyncResult< ShardInfo > create_shard(pg_id_t pg_owner, uint64_t size_bytes, std::string meta,
194+
trace_id_t tid) final;
193195
ShardManager::AsyncResult< InfoList > list_shards(pg_id_t pg, trace_id_t tid) const final;
194196
ShardManager::AsyncResult< ShardInfo > seal_shard(shard_id_t id, trace_id_t tid) final;
195197
uint64_t get_current_timestamp();

src/lib/homestore_backend/CMakeLists.txt

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ target_sources("${PROJECT_NAME}_homestore" PRIVATE
3030
hs_cp_callbacks.cpp
3131
hs_http_manager.cpp
3232
gc_manager.cpp
33+
scrub_manager.cpp
34+
MPMCPriorityQueue.hpp
3335
$<TARGET_OBJECTS:${PROJECT_NAME}_core>
3436
)
3537
target_link_libraries("${PROJECT_NAME}_homestore" PUBLIC
@@ -42,10 +44,14 @@ settings_gen_cpp(
4244
${FLATBUFFERS_FLATC_EXECUTABLE}
4345
${CMAKE_CURRENT_BINARY_DIR}/generated/
4446
"${PROJECT_NAME}_homestore"
45-
hs_backend_config.fbs
46-
resync_pg_data.fbs
47-
resync_shard_data.fbs
48-
resync_blob_data.fbs
47+
hs_homeobject_fbs/hs_backend_config.fbs
48+
hs_homeobject_fbs/resync_pg_data.fbs
49+
hs_homeobject_fbs/resync_shard_data.fbs
50+
hs_homeobject_fbs/resync_blob_data.fbs
51+
52+
hs_homeobject_fbs/scrub_common.fbs
53+
hs_homeobject_fbs/scrub_req.fbs
54+
hs_homeobject_fbs/scrub_result.fbs
4955
)
5056

5157
# Unit test objects
@@ -165,3 +171,12 @@ add_test(NAME HomestoreTestGC COMMAND homestore_test_gc -csv error --executor im
165171
--override_config hs_backend_config.gc_enable_read_verify=true
166172
--override_config hs_backend_config.gc_garbage_rate_threshold=0
167173
--override_config hs_backend_config.gc_scan_interval_sec=5)
174+
175+
add_executable(homestore_test_scrubber)
176+
target_sources(homestore_test_scrubber PRIVATE $<TARGET_OBJECTS:homestore_tests_scrubber>)
177+
target_link_libraries(homestore_test_scrubber PUBLIC homeobject_homestore ${COMMON_TEST_DEPS})
178+
add_test(NAME HomestoreTestScrubber COMMAND homestore_test_scrubber -csv error --executor immediate --config_path ./
179+
--override_config hs_backend_config.enable_scrubber=true
180+
--override_config nuraft_mesg_config.mesg_factory_config.data_request_deadline_secs:10)
181+
182+
Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
#pragma once
2+
3+
#include <condition_variable>
4+
#include <concepts>
5+
#include <cstddef>
6+
#include <functional>
7+
#include <mutex>
8+
#include <optional>
9+
#include <queue>
10+
#include <utility>
11+
#include <vector>
12+
13+
namespace homeobject {
14+
15+
/**
16+
* @brief Multi-Producer Multi-Consumer Priority Queue (C++20)
17+
*
18+
* Thread-safe priority queue that supports:
19+
* - Concurrent push operations from multiple producers
20+
* - Concurrent pop operations from multiple consumers
21+
* - Blocking pop when queue is empty
22+
* - Graceful shutdown via close() method
23+
*
24+
* @tparam T Element type (must be comparable)
25+
* @tparam Compare Comparison function (default: std::less for max-heap)
26+
*/
27+
template < typename T, typename Compare = std::less< T > >
28+
requires std::movable< T > && std::predicate< Compare, T, T >
29+
class MPMCPriorityQueue {
30+
public:
31+
using value_type = T;
32+
using size_type = std::size_t;
33+
using comparator_type = Compare;
34+
35+
/**
36+
* @brief Status codes returned by pop operations
37+
*/
38+
enum class Status : uint8_t {
39+
Ok, ///< Successfully popped an element
40+
Closed ///< Queue is closed, no more elements available
41+
};
42+
43+
/**
44+
* @brief Result of a pop operation
45+
*/
46+
struct PopResult {
47+
Status status;
48+
std::optional< T > value; ///< Has value only if status == Ok
49+
50+
// Convenience methods
51+
[[nodiscard]] constexpr bool is_ok() const noexcept { return status == Status::Ok; }
52+
[[nodiscard]] constexpr bool is_closed() const noexcept { return status == Status::Closed; }
53+
};
54+
55+
/**
56+
* @brief Construct an empty priority queue
57+
*/
58+
constexpr MPMCPriorityQueue() noexcept(std::is_nothrow_default_constructible_v< Compare >) = default;
59+
60+
/**
61+
* @brief Destructor - automatically closes the queue
62+
*/
63+
~MPMCPriorityQueue() { close(); }
64+
65+
// Disable copy and move to prevent issues with condition variables
66+
MPMCPriorityQueue(const MPMCPriorityQueue&) = delete;
67+
MPMCPriorityQueue& operator=(const MPMCPriorityQueue&) = delete;
68+
MPMCPriorityQueue(MPMCPriorityQueue&&) = delete;
69+
MPMCPriorityQueue& operator=(MPMCPriorityQueue&&) = delete;
70+
71+
/**
72+
* @brief Thread-safe push operation (copy)
73+
*
74+
* @param value Element to insert
75+
* @return true if pushed successfully, false if queue is closed
76+
*/
77+
bool push(const T& value)
78+
requires std::copy_constructible< T >
79+
{
80+
{
81+
std::scoped_lock lock(mutex_);
82+
if (closed_) [[unlikely]] {
83+
return false; // Queue is closed, cannot push
84+
}
85+
pq_.push(value);
86+
}
87+
cv_.notify_one(); // Wake one waiting consumer
88+
return true;
89+
}
90+
91+
/**
92+
* @brief Thread-safe push operation (move)
93+
*
94+
* @param value Element to insert (will be moved)
95+
* @return true if pushed successfully, false if queue is closed
96+
*/
97+
bool push(T&& value) {
98+
{
99+
std::scoped_lock lock(mutex_);
100+
if (closed_) [[unlikely]] { return false; }
101+
pq_.push(std::move(value));
102+
}
103+
cv_.notify_one();
104+
return true;
105+
}
106+
107+
/**
108+
* @brief Thread-safe pop operation
109+
*
110+
* Blocks if queue is empty and not closed.
111+
* Returns immediately if queue is closed.
112+
*
113+
* @return PopResult containing status and optional value
114+
* @note Thread-safe for multiple concurrent consumers
115+
*/
116+
[[nodiscard]] PopResult pop() {
117+
std::unique_lock lock(mutex_);
118+
119+
// Wait until queue has elements or is closed
120+
cv_.wait(lock, [this] { return closed_ || !pq_.empty(); });
121+
122+
// Try to pop an element
123+
if (!pq_.empty()) {
124+
T top = std::move(const_cast< T& >(pq_.top()));
125+
pq_.pop();
126+
return PopResult{.status = Status::Ok, .value = std::move(top)};
127+
}
128+
129+
// Queue is empty and closed
130+
return PopResult{.status = Status::Closed, .value = std::nullopt};
131+
}
132+
133+
/**
134+
* @brief Close the queue
135+
*
136+
* After calling close():
137+
* - All blocked pop() calls will wake up
138+
* - Existing elements can still be popped
139+
* - New push() calls will be ignored
140+
* - pop() returns Status::Closed when queue becomes empty
141+
*
142+
* @note Thread-safe and idempotent
143+
*/
144+
void close() noexcept {
145+
{
146+
std::scoped_lock lock(mutex_);
147+
closed_ = true;
148+
}
149+
cv_.notify_all(); // Wake all waiting consumers
150+
}
151+
152+
/**
153+
* @brief Get current number of elements
154+
*
155+
* @return Number of elements in the queue
156+
* @note Thread-safe
157+
*/
158+
[[nodiscard]] size_type size() const {
159+
std::scoped_lock lock(mutex_);
160+
return pq_.size();
161+
}
162+
163+
/**
164+
* @brief Check if queue is empty
165+
*
166+
* @return true if queue has no elements
167+
* @note Thread-safe
168+
*/
169+
[[nodiscard]] bool empty() const {
170+
std::scoped_lock lock(mutex_);
171+
return pq_.empty();
172+
}
173+
174+
/**
175+
* @brief Check if queue is closed
176+
*
177+
* @return true if close() has been called
178+
* @note Thread-safe
179+
*/
180+
[[nodiscard]] bool is_closed() const {
181+
std::scoped_lock lock(mutex_);
182+
return closed_;
183+
}
184+
185+
private:
186+
mutable std::mutex mutex_;
187+
std::condition_variable cv_;
188+
bool closed_{false};
189+
std::priority_queue< T, std::vector< T >, Compare > pq_;
190+
};
191+
192+
} // namespace homeobject

src/lib/homestore_backend/heap_chunk_selector.cpp

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -381,8 +381,6 @@ void HeapChunkSelector::switch_chunks_for_pg(const pg_id_t pg_id, const chunk_nu
381381
std::unique_lock lk(pg_chunk_collection->mtx);
382382
auto& pg_chunks = pg_chunk_collection->m_pg_chunks;
383383

384-
// LOGDEBUGMOD(homeobject, "gc: before switch chunks for pg_id={}, pg_chunks={}", pg_chunks);
385-
386384
if (sisl_unlikely(pg_chunks[v_chunk_id]->get_chunk_id() == new_chunk_id)) {
387385
// this might happens when crash recovery. the crash happens after pg metablk is updated but before gc task
388386
// metablk is destroyed.

src/lib/homestore_backend/hs_blob_manager.cpp

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,13 @@ bool HSHomeObject::local_add_blob_info(pg_id_t const pg_id, BlobInfo const& blob
247247
} else {
248248
BLOGT(tid, blob_info.shard_id, blob_info.blob_id, "blob already exists in index table, skip it.");
249249
}
250+
251+
hs_pg->last_committed_blob_id.store(blob_info.blob_id);
252+
253+
// local_add_blob_info will also be called if br happens, in this case, last_committed_blob_id will be finally
254+
// updated to the correct value after br is done, so we don't need to worry about the case where
255+
// last_committed_blob_id is updated to a smaller value than the current last_committed_blob_id
256+
250257
return true;
251258
}
252259

src/lib/homestore_backend/hs_homeobject.cpp

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,14 @@ void HSHomeObject::init_homestore() {
312312
} else {
313313
LOGI("GC is disabled");
314314
}
315+
316+
// start scrubber
317+
if (HS_BACKEND_DYNAMIC_CONFIG(enable_scrubber)) {
318+
LOGI("Starting scrub manager");
319+
scrub_mgr_->start();
320+
} else {
321+
LOGI("scrub manager is disabled");
322+
}
315323
}
316324

317325
void HSHomeObject::on_replica_restart() {
@@ -362,7 +370,6 @@ void HSHomeObject::on_replica_restart() {
362370

363371
// gc_manager will be created only once here. we need make sure gc manager is created after all the pg meta blk
364372
// are replayed since we build pdev chunk heap in the constructor of gc manager , which depends on the pg meta.
365-
366373
// gc metablk handlers are registered in the constructor of gc manager
367374
gc_mgr_ = std::make_shared< GCManager >(this);
368375

@@ -430,6 +437,9 @@ void HSHomeObject::on_replica_restart() {
430437

431438
gc_mgr_->handle_all_recovered_gc_tasks();
432439
});
440+
441+
// initialize scrub manager
442+
scrub_mgr_ = std::make_shared< ScrubManager >(this);
433443
}
434444

435445
#if 0
@@ -499,16 +509,20 @@ void HSHomeObject::shutdown() {
499509
LOGI("waiting for {} pending requests to complete", pending_reqs);
500510
std::this_thread::sleep_for(std::chrono::milliseconds(1000));
501511
};
502-
LOGI("start stopping GC");
512+
LOGI("stopping GC");
503513
// we need stop gc before shutting down homestore(where metaservice is shutdown), because gc mgr needs metaservice
504514
// to persist gc task metablk if there is any ongoing gc task. after stopping gc manager, there is no gc task
505515
// anymore, and thus now new gc task will be written to metaservice during homestore shutdown.
506-
gc_mgr_->stop();
516+
if (gc_mgr_) gc_mgr_->stop();
517+
518+
LOGI("stopping scrubbing");
519+
if (scrub_mgr_) scrub_mgr_->stop();
507520

508521
LOGI("start shutting down HomeStore");
509522
homestore::HomeStore::instance()->shutdown();
510523
homestore::HomeStore::reset_instance();
511524
gc_mgr_.reset();
525+
scrub_mgr_.reset();
512526
iomanager.stop();
513527
LOGI("complete shutting down HomeStore");
514528
}

0 commit comments

Comments
 (0)