Enable multiscan in crash test (facebook#13888)

Summary:
I ran multiple runs of crash test jobs internally, so far I've seen one iterator mismatch and one assertion failure. I've added relevant logging improvements to help debugging them. use_multiscan will be stable within a crash test run to make it easier to triage.

Pull Request resolved: facebook#13888

Test Plan: `python3 tools/db_crashtest.py whitebox --prefix_size=-1 --test_batches_snapshots=0 --use_multiscan=1 --read_fault_one_in=0 --kill_random_test=88888`

Reviewed By: anand1976

Differential Revision: D80627399

Pulled By: cbi42

fbshipit-source-id: 2fa3f77e730f5bc7d1d200dc122cf84e3558c588

Author: cbi42

db_stress_tool/db_stress_test_base.cc

@@ -1701,7 +1701,7 @@ Status StressTest::TestMultiScan(ThreadState* thread,
   iter.reset(db_->NewIterator(ro, column_families_[rand_column_families[0]]));
   iter->Prepare(scan_opts);
 
-  constexpr size_t kOpLogsLimit = 10000;
+  constexpr size_t kOpLogsLimit = 50000;
 
   auto verify_func = [](Iterator* iterator) {
     if (!VerifyWideColumns(iterator->value(), iterator->columns())) {
@@ -1801,11 +1801,24 @@ Status StressTest::TestMultiScan(ThreadState* thread,
 
       VerifyIterator(thread, cmp_cfh, ro, iter.get(), cmp_iter.get(), last_op,
                      key, op_logs, verify_func, &diverged);
+
+      if (diverged) {
+        const std::vector<ScanOptions>& scanoptions = scan_opts.GetScanRanges();
+        for (const auto& t : scanoptions) {
+          fprintf(stdout, "Multiscan options: %s to %s \n",
+                  t.range.start.value().ToString(true).c_str(),
+                  t.range.limit.value().ToString(true).c_str());
+        }
+        break;
+      }
     }
 
     thread->stats.AddIterations(1);
 
     op_logs += "; ";
+    if (diverged) {
+      break;
+    }
   }
 
   return Status::OK();

table/block_based/block_based_table_iterator.cc

@@ -1106,6 +1106,37 @@ void BlockBasedTableIterator::Prepare(const MultiScanArgs* multiscan_opts) {
       const auto start_offset = first_block.offset();
       const auto end_offset = last_block.offset() +
                               BlockBasedTable::BlockSizeWithTrailer(last_block);
+#ifndef NDEBUG
+      // Debug print for failing the assertion below.
+      if (start_offset >= end_offset) {
+        fprintf(stderr, "blocks_to_prepare: ");
+        for (const auto& block : blocks_to_prepare) {
+          fprintf(stderr, "offset: %" PRIu64 ", size: %" PRIu64 "; ",
+                  block.offset(), block.size());
+        }
+        fprintf(stderr,
+                "\nfirst block - offset: %" PRIu64 ", size: %" PRIu64 "\n",
+                first_block.offset(), first_block.size());
+        fprintf(stderr, "last block - offset: %" PRIu64 ", size: %" PRIu64 "\n",
+                last_block.offset(), last_block.size());
+
+        fprintf(stderr, "collapsed_blocks_to_read: ");
+        for (const auto& b : collapsed_blocks_to_read) {
+          fprintf(stderr, "[");
+          for (const auto& block_idx : b) {
+            fprintf(stderr, "%zu ", block_idx);
+          }
+          fprintf(stderr, "] ");
+        }
+        fprintf(stderr, "\ncurrent blocks: ");
+        for (const auto& block_idx : blocks) {
+          fprintf(stderr, "offset: %" PRIu64 ", size: %" PRIu64 "; ",
+                  blocks_to_prepare[block_idx].offset(),
+                  blocks_to_prepare[block_idx].size());
+        }
+        fprintf(stderr, "\n");
+      }
+#endif  // NDEBUG
       assert(end_offset > start_offset);
       FSReadRequest read_req;
       read_req.offset = start_offset;

tools/db_crashtest.py

@@ -181,7 +181,6 @@
     "format_version": lambda: random.choice([2, 3, 4, 5, 6, 7, 7]),
     "index_block_restart_interval": lambda: random.choice(range(1, 16)),
     "use_multiget": lambda: random.randint(0, 1),
-    "use_multiscan": 0,
     "use_get_entity": lambda: random.choice([0] * 7 + [1]),
     "use_multi_get_entity": lambda: random.choice([0] * 7 + [1]),
     "periodic_compaction_seconds": lambda: random.choice([0, 0, 1, 2, 10, 100, 1000]),
@@ -360,6 +359,9 @@
         + ["randommixed"] * 2
         + ["custom"] * 3
     ),
+    # fixed within a run for easier debugging
+    # actual frequency is lower after option sanitization
+    "use_multiscan": random.choice([1] + [0] * 3),
 }
 
 _TEST_DIR_ENV_VAR = "TEST_TMPDIR"
@@ -757,7 +759,7 @@ def finalize_and_sanitize(src_params):
     if (
         dest_params.get("test_batches_snapshots") == 1
         or dest_params.get("use_txn") == 1
-        or dest_params.get("user_timestamp_size") > 0
+        or dest_params.get("user_timestamp_size", 0) > 0
     ):
         dest_params["ingest_external_file_one_in"] = 0
     if (
@@ -785,7 +787,7 @@ def finalize_and_sanitize(src_params):
     if (
         dest_params.get("sync_fault_injection") == 1
         or dest_params.get("disable_wal") == 1
-        or dest_params.get("manual_wal_flush_one_in") > 0
+        or dest_params.get("manual_wal_flush_one_in", 0) > 0
     ):
         # File ingestion does not guarantee prefix-recoverability when unsynced
         # data can be lost. Ingesting a file syncs data immediately that is
@@ -992,7 +994,7 @@ def finalize_and_sanitize(src_params):
         dest_params["check_multiget_entity_consistency"] = 0
     if dest_params.get("disable_wal") == 0:
         if (
-            dest_params.get("reopen") > 0
+            dest_params.get("reopen", 0) > 0
             or (
                 dest_params.get("manual_wal_flush_one_in")
                 and dest_params.get("column_families") != 1
@@ -1061,7 +1063,7 @@ def finalize_and_sanitize(src_params):
     if dest_params.get("use_put_entity_one_in") == 1:
         dest_params["use_timed_put_one_in"] = 0
     elif (
-        dest_params.get("use_put_entity_one_in") > 1
+        dest_params.get("use_put_entity_one_in", 0) > 1
         and dest_params.get("use_timed_put_one_in") == 1
     ):
         dest_params["use_timed_put_one_in"] = 3
@@ -1102,6 +1104,14 @@ def finalize_and_sanitize(src_params):
     # Continuous verification fails with secondaries inside NonBatchedOpsStressTest
     if dest_params.get("test_secondary") == 1:
         dest_params["continuous_verification_interval"] = 0
+    if (
+        dest_params.get("prefix_size", 0) > 0
+        or dest_params.get("read_fault_one_in", 0) > 0
+    ):
+        dest_params["use_multiscan"] = 0
+    if dest_params.get("use_multiscan") == 1:
+        dest_params["fill_cache"] = 1
+        dest_params["async_io"] = 0
     return dest_params