Skip to content

Commit b040ab8

Browse files
xingbowangmeta-codesync[bot]
authored andcommitted
Add a new picking algorithm in fifo compaction (facebook#14326)
Summary: Add a new kv ratio based compaction picking algorithm in fifo compaction Pull Request resolved: facebook#14326 Test Plan: Unit test Reviewed By: pdillinger Differential Revision: D93257941 Pulled By: xingbowang fbshipit-source-id: fd2d0e1356c7b54682a1197475a1bd26cb45c9d4
1 parent 9f47518 commit b040ab8

24 files changed

Lines changed: 2442 additions & 103 deletions

db/c.cc

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7047,6 +7047,27 @@ uint64_t rocksdb_fifo_compaction_options_get_max_table_files_size(
70477047
return fifo_opts->rep.max_table_files_size;
70487048
}
70497049

7050+
void rocksdb_fifo_compaction_options_set_max_data_files_size(
7051+
rocksdb_fifo_compaction_options_t* fifo_opts, uint64_t size) {
7052+
fifo_opts->rep.max_data_files_size = size;
7053+
}
7054+
7055+
uint64_t rocksdb_fifo_compaction_options_get_max_data_files_size(
7056+
rocksdb_fifo_compaction_options_t* fifo_opts) {
7057+
return fifo_opts->rep.max_data_files_size;
7058+
}
7059+
7060+
void rocksdb_fifo_compaction_options_set_use_kv_ratio_compaction(
7061+
rocksdb_fifo_compaction_options_t* fifo_opts,
7062+
unsigned char use_kv_ratio_compaction) {
7063+
fifo_opts->rep.use_kv_ratio_compaction = use_kv_ratio_compaction;
7064+
}
7065+
7066+
unsigned char rocksdb_fifo_compaction_options_get_use_kv_ratio_compaction(
7067+
rocksdb_fifo_compaction_options_t* fifo_opts) {
7068+
return fifo_opts->rep.use_kv_ratio_compaction;
7069+
}
7070+
70507071
void rocksdb_fifo_compaction_options_destroy(
70517072
rocksdb_fifo_compaction_options_t* fifo_opts) {
70527073
delete fifo_opts;

db/c_test.c

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3596,6 +3596,14 @@ int main(int argc, char** argv) {
35963596
100000 ==
35973597
rocksdb_fifo_compaction_options_get_max_table_files_size(fco));
35983598

3599+
rocksdb_fifo_compaction_options_set_max_data_files_size(fco, 200000);
3600+
CheckCondition(
3601+
200000 == rocksdb_fifo_compaction_options_get_max_data_files_size(fco));
3602+
3603+
rocksdb_fifo_compaction_options_set_use_kv_ratio_compaction(fco, 1);
3604+
CheckCondition(
3605+
1 == rocksdb_fifo_compaction_options_get_use_kv_ratio_compaction(fco));
3606+
35993607
rocksdb_fifo_compaction_options_destroy(fco);
36003608
}
36013609

db/column_family.cc

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -401,7 +401,13 @@ ColumnFamilyOptions SanitizeCfOptions(const ImmutableDBOptions& db_options,
401401
}
402402

403403
if (result.max_compaction_bytes == 0) {
404-
result.max_compaction_bytes = result.target_file_size_base * 25;
404+
// For FIFO with use_kv_ratio_compaction, leave max_compaction_bytes as 0
405+
// to signal "auto-calculate target from capacity and SST/blob ratio."
406+
// When explicitly set by the user, it overrides the auto-calculated target.
407+
if (result.compaction_style != kCompactionStyleFIFO ||
408+
!result.compaction_options_fifo.use_kv_ratio_compaction) {
409+
result.max_compaction_bytes = result.target_file_size_base * 25;
410+
}
405411
}
406412

407413
bool is_block_based_table = (result.table_factory->IsInstanceOf(
@@ -1563,6 +1569,34 @@ Status ColumnFamilyData::ValidateOptions(
15631569
"FIFO compaction only supported with max_open_files = -1.");
15641570
}
15651571

1572+
if (cf_options.compaction_options_fifo.use_kv_ratio_compaction) {
1573+
if (cf_options.compaction_style != kCompactionStyleFIFO) {
1574+
return Status::InvalidArgument(
1575+
"use_kv_ratio_compaction is only supported with FIFO compaction "
1576+
"style.");
1577+
}
1578+
if (!cf_options.compaction_options_fifo.allow_compaction) {
1579+
return Status::InvalidArgument(
1580+
"use_kv_ratio_compaction requires allow_compaction = true. "
1581+
"allow_compaction enables intra-L0 compaction, and "
1582+
"use_kv_ratio_compaction selects the picking strategy.");
1583+
}
1584+
if (cf_options.compaction_options_fifo.max_data_files_size == 0) {
1585+
return Status::InvalidArgument(
1586+
"use_kv_ratio_compaction requires max_data_files_size > 0 to "
1587+
"compute the target compacted file size from data capacity.");
1588+
}
1589+
}
1590+
1591+
if (cf_options.compaction_options_fifo.max_data_files_size > 0 &&
1592+
cf_options.compaction_options_fifo.max_data_files_size <
1593+
cf_options.compaction_options_fifo.max_table_files_size) {
1594+
return Status::InvalidArgument(
1595+
"max_data_files_size (total data = SST + blob) must be >= "
1596+
"max_table_files_size (SST only) when non-zero, since total data "
1597+
"always includes SST data.");
1598+
}
1599+
15661600
std::vector<uint32_t> supported{0, 1, 2, 4, 8};
15671601
if (std::find(supported.begin(), supported.end(),
15681602
cf_options.memtable_protection_bytes_per_key) ==

db/compaction/compaction_picker.cc

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,11 @@
2727

2828
namespace ROCKSDB_NAMESPACE {
2929

30-
bool FindIntraL0Compaction(const std::vector<FileMetaData*>& level_files,
31-
size_t min_files_to_compact,
32-
uint64_t max_compact_bytes_per_del_file,
33-
uint64_t max_compaction_bytes,
34-
CompactionInputFiles* comp_inputs) {
35-
TEST_SYNC_POINT("FindIntraL0Compaction");
30+
bool PickCostBasedIntraL0Compaction(
31+
const std::vector<FileMetaData*>& level_files, size_t min_files_to_compact,
32+
uint64_t max_compact_bytes_per_del_file, uint64_t max_compaction_bytes,
33+
CompactionInputFiles* comp_inputs) {
34+
TEST_SYNC_POINT("PickCostBasedIntraL0Compaction");
3635

3736
size_t start = 0;
3837

db/compaction/compaction_picker.h

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -328,11 +328,10 @@ class NullCompactionPicker : public CompactionPicker {
328328
// files. Cannot be nullptr.
329329
//
330330
// @return true iff compaction was found.
331-
bool FindIntraL0Compaction(const std::vector<FileMetaData*>& level_files,
332-
size_t min_files_to_compact,
333-
uint64_t max_compact_bytes_per_del_file,
334-
uint64_t max_compaction_bytes,
335-
CompactionInputFiles* comp_inputs);
331+
bool PickCostBasedIntraL0Compaction(
332+
const std::vector<FileMetaData*>& level_files, size_t min_files_to_compact,
333+
uint64_t max_compact_bytes_per_del_file, uint64_t max_compaction_bytes,
334+
CompactionInputFiles* comp_inputs);
336335

337336
CompressionType GetCompressionType(const VersionStorageInfo* vstorage,
338337
const MutableCFOptions& mutable_cf_options,

0 commit comments

Comments
 (0)