Add MultiScan statistics (facebook#14248)

Summary:
Pull Request resolved: facebook#14248

### Overview

This diff introduces the addition of multi-scan statistics to RocksDB, enhancing the database's ability to monitor and analyze performance during multi-scan operations.

### Key Changes

#### Implemented Multi-Scan Statistics

The following statistics were implemented to provide deeper insights into multi-scan operations:

- **MULTISCAN_PREPARE_MICROS**: Measures the time (in microseconds) spent preparing for multi-scan operations.
- **MULTISCAN_BLOCKS_PER_PREPARE**: Tracks the number of blocks processed per multi-scan prepare operation.
- **Wasted Prefetch Blocks Count**: Counts the number of prefetched blocks that were not used (i.e., wasted) if the iterator is abandoned before accessing them.
- **MULTISCAN_TOTAL_BLOCKS_SCANNED**: Tracks the total number of blocks scanned during all multi-scan operations.
- **MULTISCAN_TOTAL_KEYS_SCANNED**: Measures the total number of keys scanned across all multi-scan operations.
- **MULTISCAN_TOTAL_MICROS**: Captures the total time (in microseconds) spent in multi-scan operations.
- **MULTISCAN_PREFETCHED_BLOCKS**: Counts the number of blocks that were prefetched during multi-scan operations.
- **MULTISCAN_USED_PREFETCH_BLOCKS**: Tracks the number of prefetched blocks that were actually used during multi-scan operations.

### Impact

This diff provides more fine-grained statistics for multi-scan operations, allowing developers and users to better understand and optimize the performance of their RocksDB instances.

Reviewed By: krhancoc

Differential Revision: D91053297

fbshipit-source-id: 7158741b9f026c0b5ce8ba1264dbd137e7fe985d

Author: anand1976

db/db_iterator_test.cc

@@ -5027,6 +5027,152 @@ TEST_P(DBMultiScanIteratorTest, AsyncPrefetchWithExternalFileIngestion) {
   ASSERT_EQ(total_keys, 400);
   iter.reset();
 }
+
+TEST_P(DBMultiScanIteratorTest, StatisticsTest) {
+  // Test that multi scan statistics are properly recorded
+  auto options = CurrentOptions();
+  options.statistics = CreateDBStatistics();
+  // Use small block size to ensure multiple blocks
+  BlockBasedTableOptions table_options;
+  table_options.block_size = 256;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  DestroyAndReopen(options);
+
+  // Create data across multiple blocks
+  for (int i = 0; i < 100; ++i) {
+    std::stringstream ss;
+    ss << std::setw(3) << std::setfill('0') << i;
+    // Use larger values to ensure multiple blocks
+    ASSERT_OK(Put("k" + ss.str(), std::string(100, 'v')));
+  }
+  ASSERT_OK(Flush());
+
+  // Reset stats before multi scan
+  ASSERT_OK(options.statistics->Reset());
+
+  // Set up two scan ranges
+  std::vector<std::string> key_ranges({"k010", "k030", "k060", "k080"});
+  ReadOptions ro;
+  ro.fill_cache = GetParam();
+  MultiScanArgs scan_options(BytewiseComparator());
+  scan_options.insert(key_ranges[0], key_ranges[1]);
+  scan_options.insert(key_ranges[2], key_ranges[3]);
+
+  ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily();
+  std::unique_ptr<MultiScan> iter =
+      dbfull()->NewMultiScan(ro, cfh, scan_options);
+
+  // Iterate through all ranges
+  int count = 0;
+  try {
+    for (auto range : *iter) {
+      for (auto it : range) {
+        (void)it;
+        count++;
+      }
+    }
+  } catch (MultiScanException& ex) {
+    ASSERT_NOK(ex.status());
+    std::cerr << "Iterator returned status " << ex.what();
+    abort();
+  }
+  ASSERT_EQ(count, 40);  // 20 keys per range
+  iter.reset();
+
+  // Check statistics
+  // MULTISCAN_PREPARE_CALLS should be at least 1
+  ASSERT_GE(TestGetTickerCount(options, MULTISCAN_PREPARE_CALLS), 1);
+
+  // MULTISCAN_PREPARE_ERRORS should be 0
+  ASSERT_EQ(TestGetTickerCount(options, MULTISCAN_PREPARE_ERRORS), 0);
+
+  // MULTISCAN_SEEK_ERRORS should be 0
+  ASSERT_EQ(TestGetTickerCount(options, MULTISCAN_SEEK_ERRORS), 0);
+
+  // Blocks should be prefetched or from cache
+  uint64_t blocks_prefetched =
+      TestGetTickerCount(options, MULTISCAN_BLOCKS_PREFETCHED);
+  uint64_t blocks_from_cache =
+      TestGetTickerCount(options, MULTISCAN_BLOCKS_FROM_CACHE);
+  ASSERT_GT(blocks_prefetched + blocks_from_cache, 0);
+
+  // If blocks were prefetched, prefetch bytes and IO requests should be > 0
+  if (blocks_prefetched > 0) {
+    ASSERT_GT(TestGetTickerCount(options, MULTISCAN_PREFETCH_BYTES), 0);
+    uint64_t io_requests = TestGetTickerCount(options, MULTISCAN_IO_REQUESTS);
+    ASSERT_GT(io_requests, 0);
+    ASSERT_LE(io_requests, blocks_prefetched);
+  }
+
+  // Wasted blocks should be 0 since we iterated through everything
+  ASSERT_EQ(TestGetTickerCount(options, MULTISCAN_PREFETCH_BLOCKS_WASTED), 0);
+}
+
+TEST_P(DBMultiScanIteratorTest, StatisticsWastedBlocksTest) {
+  // Test that wasted blocks are tracked when iteration is abandoned early
+  auto options = CurrentOptions();
+  options.statistics = CreateDBStatistics();
+  // Use small block size to ensure multiple blocks
+  BlockBasedTableOptions table_options;
+  table_options.block_size = 256;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  DestroyAndReopen(options);
+
+  // Create data across multiple blocks
+  for (int i = 0; i < 100; ++i) {
+    std::stringstream ss;
+    ss << std::setw(3) << std::setfill('0') << i;
+    ASSERT_OK(Put("k" + ss.str(), std::string(100, 'v')));
+  }
+  ASSERT_OK(Flush());
+
+  // Reset stats before multi scan
+  ASSERT_OK(options.statistics->Reset());
+
+  // Set up a large scan range
+  ReadOptions ro;
+  ro.fill_cache = GetParam();
+  MultiScanArgs scan_options(BytewiseComparator());
+  scan_options.insert("k000", "k099");
+
+  ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily();
+  std::unique_ptr<MultiScan> iter =
+      dbfull()->NewMultiScan(ro, cfh, scan_options);
+
+  // Only iterate through a few keys, then abandon
+  int count = 0;
+  try {
+    for (auto range : *iter) {
+      for (auto it : range) {
+        (void)it;
+        count++;
+        if (count >= 5) {
+          break;  // Abandon iteration early
+        }
+      }
+      if (count >= 5) {
+        break;
+      }
+    }
+  } catch (MultiScanException& ex) {
+    ASSERT_NOK(ex.status());
+    std::cerr << "Iterator returned status " << ex.what();
+    abort();
+  }
+  ASSERT_EQ(count, 5);
+
+  // Destroy iterator to trigger wasted blocks counting
+  iter.reset();
+
+  uint64_t blocks_prefetched =
+      TestGetTickerCount(options, MULTISCAN_BLOCKS_PREFETCHED);
+
+  // If blocks were prefetched, some should be wasted since we abandoned early
+  if (blocks_prefetched > 1) {
+    // We only read a few keys, so there should be wasted blocks
+    ASSERT_GT(TestGetTickerCount(options, MULTISCAN_PREFETCH_BLOCKS_WASTED), 0);
+  }
+}
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

include/rocksdb/statistics.h

@@ -552,6 +552,27 @@ enum Tickers : uint32_t {
   // Failure to load the UDI during SST table open
   SST_USER_DEFINED_INDEX_LOAD_FAIL_COUNT,
 
+  // MultiScan statistics
+  // # of Prepare() calls
+  MULTISCAN_PREPARE_CALLS,
+  // # of Prepare() calls that failed
+  MULTISCAN_PREPARE_ERRORS,
+  // # of data blocks prefetched from storage during MultiScan
+  MULTISCAN_BLOCKS_PREFETCHED,
+  // # of blocks found already in cache during MultiScan Prepare
+  MULTISCAN_BLOCKS_FROM_CACHE,
+  // Total bytes prefetched during MultiScan
+  MULTISCAN_PREFETCH_BYTES,
+  // # of prefetched blocks that were never accessed
+  MULTISCAN_PREFETCH_BLOCKS_WASTED,
+  // # of actual I/O requests issued during MultiScan
+  MULTISCAN_IO_REQUESTS,
+  // # of non-adjacent blocks coalesced into single I/O (within
+  // io_coalesce_threshold)
+  MULTISCAN_IO_COALESCED_NONADJACENT,
+  // # of seeks that failed validation (out of order, etc.)
+  MULTISCAN_SEEK_ERRORS,
+
   TICKER_ENUM_MAX
 };
 
@@ -695,6 +716,11 @@ enum Histograms : uint32_t {
   // MultiScan Prefill iterator Prepare cost
   MULTISCAN_PREPARE_ITERATORS,
 
+  // Total Prepare() latency for MultiScan
+  MULTISCAN_PREPARE_MICROS,
+  // Distribution of blocks prefetched per MultiScan Prepare()
+  MULTISCAN_BLOCKS_PER_PREPARE,
+
   HISTOGRAM_ENUM_MAX
 };
 

java/rocksjni/portal.h

@@ -5289,6 +5289,24 @@ class TickerTypeJni {
         return -0x5D;
       case ROCKSDB_NAMESPACE::Tickers::SST_USER_DEFINED_INDEX_LOAD_FAIL_COUNT:
         return -0x5E;
+      case ROCKSDB_NAMESPACE::Tickers::MULTISCAN_PREPARE_CALLS:
+        return -0x60;
+      case ROCKSDB_NAMESPACE::Tickers::MULTISCAN_PREPARE_ERRORS:
+        return -0x61;
+      case ROCKSDB_NAMESPACE::Tickers::MULTISCAN_BLOCKS_PREFETCHED:
+        return -0x62;
+      case ROCKSDB_NAMESPACE::Tickers::MULTISCAN_BLOCKS_FROM_CACHE:
+        return -0x63;
+      case ROCKSDB_NAMESPACE::Tickers::MULTISCAN_PREFETCH_BYTES:
+        return -0x64;
+      case ROCKSDB_NAMESPACE::Tickers::MULTISCAN_PREFETCH_BLOCKS_WASTED:
+        return -0x65;
+      case ROCKSDB_NAMESPACE::Tickers::MULTISCAN_IO_REQUESTS:
+        return -0x66;
+      case ROCKSDB_NAMESPACE::Tickers::MULTISCAN_IO_COALESCED_NONADJACENT:
+        return -0x67;
+      case ROCKSDB_NAMESPACE::Tickers::MULTISCAN_SEEK_ERRORS:
+        return -0x68;
       case ROCKSDB_NAMESPACE::Tickers::TICKER_ENUM_MAX:
         // -0x54 is the max value at this time. Since these values are exposed
         // directly to Java clients, we'll keep the value the same till the next
@@ -5768,6 +5786,24 @@ class TickerTypeJni {
       case -0x5E:
         return ROCKSDB_NAMESPACE::Tickers::
             SST_USER_DEFINED_INDEX_LOAD_FAIL_COUNT;
+      case -0x60:
+        return ROCKSDB_NAMESPACE::Tickers::MULTISCAN_PREPARE_CALLS;
+      case -0x61:
+        return ROCKSDB_NAMESPACE::Tickers::MULTISCAN_PREPARE_ERRORS;
+      case -0x62:
+        return ROCKSDB_NAMESPACE::Tickers::MULTISCAN_BLOCKS_PREFETCHED;
+      case -0x63:
+        return ROCKSDB_NAMESPACE::Tickers::MULTISCAN_BLOCKS_FROM_CACHE;
+      case -0x64:
+        return ROCKSDB_NAMESPACE::Tickers::MULTISCAN_PREFETCH_BYTES;
+      case -0x65:
+        return ROCKSDB_NAMESPACE::Tickers::MULTISCAN_PREFETCH_BLOCKS_WASTED;
+      case -0x66:
+        return ROCKSDB_NAMESPACE::Tickers::MULTISCAN_IO_REQUESTS;
+      case -0x67:
+        return ROCKSDB_NAMESPACE::Tickers::MULTISCAN_IO_COALESCED_NONADJACENT;
+      case -0x68:
+        return ROCKSDB_NAMESPACE::Tickers::MULTISCAN_SEEK_ERRORS;
       case -0x54:
         // -0x54 is the max value at this time. Since these values are exposed
         // directly to Java clients, we'll keep the value the same till the next
@@ -5924,6 +5960,10 @@ class HistogramTypeJni {
         return 0x3D;
       case ROCKSDB_NAMESPACE::Histograms::COMPACTION_PREFETCH_BYTES:
         return 0x3F;
+      case ROCKSDB_NAMESPACE::Histograms::MULTISCAN_PREPARE_MICROS:
+        return 0x40;
+      case ROCKSDB_NAMESPACE::Histograms::MULTISCAN_BLOCKS_PER_PREPARE:
+        return 0x41;
       case ROCKSDB_NAMESPACE::Histograms::HISTOGRAM_ENUM_MAX:
         // 0x3E is reserved for backwards compatibility on current minor
         // version.
@@ -6071,6 +6111,10 @@ class HistogramTypeJni {
             TABLE_OPEN_PREFETCH_TAIL_READ_BYTES;
       case 0x3F:
         return ROCKSDB_NAMESPACE::Histograms::COMPACTION_PREFETCH_BYTES;
+      case 0x40:
+        return ROCKSDB_NAMESPACE::Histograms::MULTISCAN_PREPARE_MICROS;
+      case 0x41:
+        return ROCKSDB_NAMESPACE::Histograms::MULTISCAN_BLOCKS_PER_PREPARE;
       case 0x3E:
         // 0x3E is reserved for backwards compatibility on current minor
         // version.

java/src/main/java/org/rocksdb/HistogramType.java

@@ -212,6 +212,20 @@ public enum HistogramType {
 
   COMPACTION_PREFETCH_BYTES((byte) 0x3F),
 
+  /**
+   * MultiScan histogram statistics
+   */
+
+  /**
+   * Time spent in Iterator::Prepare() for multi-scan (microseconds)
+   */
+  MULTISCAN_PREPARE_MICROS((byte) 0x40),
+
+  /**
+   * Number of blocks per multi-scan Prepare() call
+   */
+  MULTISCAN_BLOCKS_PER_PREPARE((byte) 0x41),
+
   // 0x3E is reserved for backwards compatibility on current minor version.
   HISTOGRAM_ENUM_MAX((byte) 0x3E);
 

java/src/main/java/org/rocksdb/TickerType.java

@@ -906,6 +906,55 @@ public enum TickerType {
      */
     REMOTE_COMPACT_RESUMED_BYTES((byte) -0x5F),
 
+    /**
+     * MultiScan statistics
+     */
+
+    /**
+     * # of calls to Iterator::Prepare() for multi-scan
+     */
+    MULTISCAN_PREPARE_CALLS((byte) -0x60),
+
+    /**
+     * # of errors during Iterator::Prepare() for multi-scan
+     */
+    MULTISCAN_PREPARE_ERRORS((byte) -0x61),
+
+    /**
+     * # of data blocks prefetched during multi-scan Prepare()
+     */
+    MULTISCAN_BLOCKS_PREFETCHED((byte) -0x62),
+
+    /**
+     * # of data blocks found in cache during multi-scan Prepare()
+     */
+    MULTISCAN_BLOCKS_FROM_CACHE((byte) -0x63),
+
+    /**
+     * Total bytes prefetched during multi-scan Prepare()
+     */
+    MULTISCAN_PREFETCH_BYTES((byte) -0x64),
+
+    /**
+     * # of prefetched blocks that were never accessed (wasted)
+     */
+    MULTISCAN_PREFETCH_BLOCKS_WASTED((byte) -0x65),
+
+    /**
+     * # of I/O requests issued during multi-scan Prepare()
+     */
+    MULTISCAN_IO_REQUESTS((byte) -0x66),
+
+    /**
+     * # of non-adjacent blocks coalesced into single I/O request
+     */
+    MULTISCAN_IO_COALESCED_NONADJACENT((byte) -0x67),
+
+    /**
+     * # of seek errors during multi-scan iteration
+     */
+    MULTISCAN_SEEK_ERRORS((byte) -0x68),
+
     TICKER_ENUM_MAX((byte) -0x54);
 
     private final byte value;

monitoring/statistics.cc

@@ -280,6 +280,17 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
     {NUMBER_WBWI_INGEST, "rocksdb.number.wbwi.ingest"},
     {SST_USER_DEFINED_INDEX_LOAD_FAIL_COUNT,
      "rocksdb.sst.user.defined.index.load.fail.count"},
+    {MULTISCAN_PREPARE_CALLS, "rocksdb.multiscan.prepare.calls"},
+    {MULTISCAN_PREPARE_ERRORS, "rocksdb.multiscan.prepare.errors"},
+    {MULTISCAN_BLOCKS_PREFETCHED, "rocksdb.multiscan.blocks.prefetched"},
+    {MULTISCAN_BLOCKS_FROM_CACHE, "rocksdb.multiscan.blocks.from.cache"},
+    {MULTISCAN_PREFETCH_BYTES, "rocksdb.multiscan.prefetch.bytes"},
+    {MULTISCAN_PREFETCH_BLOCKS_WASTED,
+     "rocksdb.multiscan.prefetch.blocks.wasted"},
+    {MULTISCAN_IO_REQUESTS, "rocksdb.multiscan.io.requests"},
+    {MULTISCAN_IO_COALESCED_NONADJACENT,
+     "rocksdb.multiscan.io.coalesced.nonadjacent"},
+    {MULTISCAN_SEEK_ERRORS, "rocksdb.multiscan.seek.errors"},
 };
 
 const std::vector<std::pair<Histograms, std::string>> HistogramsNameMap = {
@@ -354,6 +365,8 @@ const std::vector<std::pair<Histograms, std::string>> HistogramsNameMap = {
     {NUM_OP_PER_TRANSACTION, "rocksdb.num.op.per.transaction"},
     {MULTISCAN_PREPARE_ITERATORS,
      "rocksdb.multiscan.op.prepare.iterators.micros"},
+    {MULTISCAN_PREPARE_MICROS, "rocksdb.multiscan.prepare.micros"},
+    {MULTISCAN_BLOCKS_PER_PREPARE, "rocksdb.multiscan.blocks.per.prepare"},
 };
 
 std::shared_ptr<Statistics> CreateDBStatistics() {

monitoring/stats_history_test.cc

@@ -185,7 +185,7 @@ TEST_F(StatsHistoryTest, GetStatsHistoryInMemory) {
 
 TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) {
   constexpr int kPeriodSec = 1;
-  constexpr int kEstimatedOneSliceSize = 16000;
+  constexpr int kEstimatedOneSliceSize = 22000;
 
   Options options;
   options.create_if_missing = true;
@@ -277,7 +277,7 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) {
   // If `slice_count == 0` when new statistics are added, consider increasing
   // `kEstimatedOneSliceSize`
   ASSERT_EQ(slice_count, 1);
-  ASSERT_TRUE(stats_history_size_reopen < 16000 &&
+  ASSERT_TRUE(stats_history_size_reopen < kEstimatedOneSliceSize &&
               stats_history_size_reopen > 0);
   ASSERT_TRUE(stats_count_reopen < stats_count && stats_count_reopen > 0);
   Close();