Add a new picking algorithm in fifo compaction (facebook#14326)

Summary:
Add a new kv ratio based compaction picking algorithm in fifo compaction

Pull Request resolved: facebook#14326

Test Plan: Unit test

Reviewed By: pdillinger

Differential Revision: D93257941

Pulled By: xingbowang

fbshipit-source-id: fd2d0e1356c7b54682a1197475a1bd26cb45c9d4

Author: xingbowang

db/c.cc

@@ -7047,6 +7047,27 @@ uint64_t rocksdb_fifo_compaction_options_get_max_table_files_size(
   return fifo_opts->rep.max_table_files_size;
 }
 
+void rocksdb_fifo_compaction_options_set_max_data_files_size(
+    rocksdb_fifo_compaction_options_t* fifo_opts, uint64_t size) {
+  fifo_opts->rep.max_data_files_size = size;
+}
+
+uint64_t rocksdb_fifo_compaction_options_get_max_data_files_size(
+    rocksdb_fifo_compaction_options_t* fifo_opts) {
+  return fifo_opts->rep.max_data_files_size;
+}
+
+void rocksdb_fifo_compaction_options_set_use_kv_ratio_compaction(
+    rocksdb_fifo_compaction_options_t* fifo_opts,
+    unsigned char use_kv_ratio_compaction) {
+  fifo_opts->rep.use_kv_ratio_compaction = use_kv_ratio_compaction;
+}
+
+unsigned char rocksdb_fifo_compaction_options_get_use_kv_ratio_compaction(
+    rocksdb_fifo_compaction_options_t* fifo_opts) {
+  return fifo_opts->rep.use_kv_ratio_compaction;
+}
+
 void rocksdb_fifo_compaction_options_destroy(
     rocksdb_fifo_compaction_options_t* fifo_opts) {
   delete fifo_opts;

db/c_test.c

@@ -3596,6 +3596,14 @@ int main(int argc, char** argv) {
         100000 ==
         rocksdb_fifo_compaction_options_get_max_table_files_size(fco));
 
+    rocksdb_fifo_compaction_options_set_max_data_files_size(fco, 200000);
+    CheckCondition(
+        200000 == rocksdb_fifo_compaction_options_get_max_data_files_size(fco));
+
+    rocksdb_fifo_compaction_options_set_use_kv_ratio_compaction(fco, 1);
+    CheckCondition(
+        1 == rocksdb_fifo_compaction_options_get_use_kv_ratio_compaction(fco));
+
     rocksdb_fifo_compaction_options_destroy(fco);
   }
 

db/column_family.cc

@@ -401,7 +401,13 @@ ColumnFamilyOptions SanitizeCfOptions(const ImmutableDBOptions& db_options,
   }
 
   if (result.max_compaction_bytes == 0) {
-    result.max_compaction_bytes = result.target_file_size_base * 25;
+    // For FIFO with use_kv_ratio_compaction, leave max_compaction_bytes as 0
+    // to signal "auto-calculate target from capacity and SST/blob ratio."
+    // When explicitly set by the user, it overrides the auto-calculated target.
+    if (result.compaction_style != kCompactionStyleFIFO ||
+        !result.compaction_options_fifo.use_kv_ratio_compaction) {
+      result.max_compaction_bytes = result.target_file_size_base * 25;
+    }
   }
 
   bool is_block_based_table = (result.table_factory->IsInstanceOf(
@@ -1563,6 +1569,34 @@ Status ColumnFamilyData::ValidateOptions(
         "FIFO compaction only supported with max_open_files = -1.");
   }
 
+  if (cf_options.compaction_options_fifo.use_kv_ratio_compaction) {
+    if (cf_options.compaction_style != kCompactionStyleFIFO) {
+      return Status::InvalidArgument(
+          "use_kv_ratio_compaction is only supported with FIFO compaction "
+          "style.");
+    }
+    if (!cf_options.compaction_options_fifo.allow_compaction) {
+      return Status::InvalidArgument(
+          "use_kv_ratio_compaction requires allow_compaction = true. "
+          "allow_compaction enables intra-L0 compaction, and "
+          "use_kv_ratio_compaction selects the picking strategy.");
+    }
+    if (cf_options.compaction_options_fifo.max_data_files_size == 0) {
+      return Status::InvalidArgument(
+          "use_kv_ratio_compaction requires max_data_files_size > 0 to "
+          "compute the target compacted file size from data capacity.");
+    }
+  }
+
+  if (cf_options.compaction_options_fifo.max_data_files_size > 0 &&
+      cf_options.compaction_options_fifo.max_data_files_size <
+          cf_options.compaction_options_fifo.max_table_files_size) {
+    return Status::InvalidArgument(
+        "max_data_files_size (total data = SST + blob) must be >= "
+        "max_table_files_size (SST only) when non-zero, since total data "
+        "always includes SST data.");
+  }
+
   std::vector<uint32_t> supported{0, 1, 2, 4, 8};
   if (std::find(supported.begin(), supported.end(),
                 cf_options.memtable_protection_bytes_per_key) ==

db/compaction/compaction_picker.cc

@@ -27,12 +27,11 @@
 
 namespace ROCKSDB_NAMESPACE {
 
-bool FindIntraL0Compaction(const std::vector<FileMetaData*>& level_files,
-                           size_t min_files_to_compact,
-                           uint64_t max_compact_bytes_per_del_file,
-                           uint64_t max_compaction_bytes,
-                           CompactionInputFiles* comp_inputs) {
-  TEST_SYNC_POINT("FindIntraL0Compaction");
+bool PickCostBasedIntraL0Compaction(
+    const std::vector<FileMetaData*>& level_files, size_t min_files_to_compact,
+    uint64_t max_compact_bytes_per_del_file, uint64_t max_compaction_bytes,
+    CompactionInputFiles* comp_inputs) {
+  TEST_SYNC_POINT("PickCostBasedIntraL0Compaction");
 
   size_t start = 0;
 

db/compaction/compaction_picker.h

@@ -328,11 +328,10 @@ class NullCompactionPicker : public CompactionPicker {
 //                                        files. Cannot be nullptr.
 //
 // @return                                true iff compaction was found.
-bool FindIntraL0Compaction(const std::vector<FileMetaData*>& level_files,
-                           size_t min_files_to_compact,
-                           uint64_t max_compact_bytes_per_del_file,
-                           uint64_t max_compaction_bytes,
-                           CompactionInputFiles* comp_inputs);
+bool PickCostBasedIntraL0Compaction(
+    const std::vector<FileMetaData*>& level_files, size_t min_files_to_compact,
+    uint64_t max_compact_bytes_per_del_file, uint64_t max_compaction_bytes,
+    CompactionInputFiles* comp_inputs);
 
 CompressionType GetCompressionType(const VersionStorageInfo* vstorage,
                                    const MutableCFOptions& mutable_cf_options,